rework asset packer, misc fixed

build.sh

@@ -82,3 +82,4 @@ if ! [ -f build/libm3d.a ]; then
 	ar rcs $lib $obj
 	rm $obj
 fi
+

dub.json

@@ -13,7 +13,7 @@
 			"sourcePaths": ["src/gears", "src/shared", "src/generated", "external/xxhash", "external/dplug/math", "external/inteli"],
 			"libs-linux": ["xcb", "X11", "X11-xcb", "vulkan", "stdc++"],
 			"libs-windows": [],
-			"preGenerateCommands-linux": ["./build-vma.sh", "build/Codegen", "dub main:packer"],
+			"preGenerateCommands-linux": ["./build.sh", "build/Packer"],
 			"preGenerateCommands-windows": [],
 			"dflags-dmd": ["-P=-DSTBI_NO_SIMD"]
 		},
@@ -25,8 +25,8 @@
 			"importPaths": ["src/packer", "src/shared", "src/generated", "external/xxhash", "external/dplug/math", "external/inteli"],
 			"sourcePaths": ["src/packer", "src/shared", "src/generated", "external/xxhash", "external/dplug/math", "external/inteli"],
 			"sourceFiles-linux": ["build/libstb_image.a", "build/libm3d.a"],
-			"preGenerateCommands-linux": ["./build-vma.sh"],
+			"preGenerateCommands-linux": ["./build.sh"],
-			"postGenerateCommands-linux": ["build/Packer"],
+			"postGenerateCommands-linux": [],
 			"preGenerateCommands-windows": [],
 			"dflags-dmd": ["-P=-DSTBI_NO_SIMD"],
 		},
@@ -38,7 +38,7 @@
 			"importPaths": ["src/codegen", "src/shared", "external/xxhash", "external/dplug/math", "external/inteli"],
 			"sourcePaths": ["src/codegen", "src/shared", "external/xxhash", "external/dplug/math", "external/inteli"],
 			"sourceFiles-linux": ["build/libstb_image.a"],
-			"preGenerateCommands-linux": ["./build-vma.sh"],
+			"preGenerateCommands-linux": ["./build.sh"],
 			"preGenerateCommands-windows": [],
 			"versions": ["codegen"],
 			"dflags-dmd": ["-P=-DSTBI_NO_SIMD"],

src/gears/main.d

@@ -12,7 +12,7 @@ void main()
 	p.Window window = p.CreateWindow("Video Game", 1920, 1080);
 
 	r.Renderer rd = r.Init(&window);
-	//scope(exit) r.Destroy(&rd);
+	scope(exit) r.Destroy(&rd);
 
 	while (true)
 	{
@@ -21,7 +21,5 @@ void main()
 
 		r.Cycle(&rd);
 	}
-
-	writefln("exiting");
 }
 

src/gears/renderer.d

@@ -189,13 +189,13 @@ Init(p.Window* window)
 	};
 
 	GfxPipelineInfo triangle_info = {
-		vertex_shader: "shaders/triangle.vert",
+		vertex_shader: "shaders/triangle.vert.spv",
-		frag_shader: "shaders/triangle.frag",
+		frag_shader: "shaders/triangle.frag.spv",
 	};
 
 	GfxPipelineInfo ui_info = {
-		vertex_shader: "shaders/gui.vert",
+		vertex_shader: "shaders/gui.vert.spv",
-		frag_shader: "shaders/gui.frag",
+		frag_shader: "shaders/gui.frag.spv",
 		input_rate: IR.Instance,
 		input_rate_stride: UIVertex.sizeof,
 		vertex_attributes: [
@@ -207,8 +207,8 @@ Init(p.Window* window)
 	};
 
 	GfxPipelineInfo pbr_info = {
-		vertex_shader: "shaders/pbr.vert",
+		vertex_shader: "shaders/pbr.vert.spv",
-		frag_shader: "shaders/pbr.frag",
+		frag_shader: "shaders/pbr.frag.spv",
 		input_rate_stride: Vertex.sizeof,
 		vertex_attributes: [
 			{ binding: 0, location: 0, format: FMT.RGBA_F32, offset: 0 },
@@ -219,7 +219,7 @@ Init(p.Window* window)
 	};
 
 	CompPipelineInfo gradient_info = {
-		shader: "shaders/gradient.comp",
+		shader: "shaders/gradient.comp.spv",
 	};
 
 	rd.pbr_pipeline = BuildGfxPipeline(&rd, &pbr_info);
@@ -227,7 +227,9 @@ Init(p.Window* window)
 	rd.ui_pipeline = BuildGfxPipeline(&rd, &ui_info);
 	rd.compute_pipeline = BuildCompPipeline(&rd, &gradient_info);
 
-	rd.yoder = LoadModel(&rd, "models/yoda");
+	rd.yoder = LoadModel(&rd, "models/Tree01.m3d");
+
+	ReadModel(&rd, "models/Tree01.m3d");
 
 	return rd;
 }
@@ -241,9 +243,7 @@ Cycle(Renderer* rd)
 	f32 cam_x = sin(RDTSC()) * radius * 0.000000000001;
 	f32 cam_z = cos(RDTSC()) * radius * 0.000000000001;
 
-	Logf("cam_x %s cam_z %s", cam_x, cam_z);
+	rd.globals.world_matrix = Mat4(1.0);
-
-	rd.globals.world_matrix = Mat4.lookAt(Vec3(cam_x, 0.0, cam_z), Vec3(0.0, 0.0, 0.0), Vec3(0.0, 1.0, 0.0));
 
 	BeginFrame(rd);
 
@@ -299,12 +299,6 @@ CopyVertex(Vec4* dst, m3dv_t* src)
 		movups XMM0, src.x.offsetof[R8];
 		movups dst.x.offsetof[R9], XMM0;
 	}
-
-debug
-{
-	assert(dst.x == src.x && dst.y == src.y && dst.z == src.z && dst.w == src.w, "Vertex copy failed");
-	assert(!isNaN(dst.x) && !isNaN(dst.y) && !isNaN(dst.z) && !isNaN(dst.w), "Vertex contains NaN");
-}
 }
 
 pragma(inline): void
@@ -400,15 +394,45 @@ BuildCompPipeline(Renderer* rd, CompPipelineInfo* comp_info)
 	return CreateComputePipeline(&rd.vk, comp_info);
 }
 
+void
+ReadModel(Renderer* rd, string name)
+{
+	AssetInfo info = GetAssetInfo(name);
+
+	u8[] data = LoadAssetData(&rd.temp_arena, name);
+
+	m3d_t* m3d = m3d_load(data.ptr, null, null, null);
+
+	foreach(i; 0 .. m3d.numtexture)
+	{
+		const(char)[] tex_name = m3d.texture[i].name[0 .. strlen(m3d.texture[i].name)];
+		Logf("texture name: %r", tex_name);
+	}
+
+	if (m3d.numtexture == 0)
+	{
+		Log("No textures in model");
+	}
+
+	foreach(i; 0 .. m3d.nummaterial)
+	{
+		const(char)[] mat_name = m3d.material[i].name[0 .. strlen(m3d.material[i].name)];
+		Logf("material name: %r", mat_name);
+	}
+}
+
 Model
 LoadModel(Renderer* rd, string name)
 {
+	AssetInfo info = GetAssetInfo(name);
+
+	u8[] data = LoadAssetData(&rd.temp_arena, name);
 	Model model = {
 		name: name,
 	};
 
-	u8[] data = LoadAssetData(&rd.temp_arena, name);
 	m3d_t* m3d = m3d_load(data.ptr, null, null, null);
+	scope(exit) m3d_free(m3d);
 
 	u32[] tex_lookup = AllocArray!(u32)(&rd.temp_arena, m3d.numtexture);
 	model.textures = AllocArray!(ImageView)(&rd.arena, m3d.numtexture);
@@ -417,6 +441,8 @@ LoadModel(Renderer* rd, string name)
 		u32 w = m3d.texture[i].w; u32 h = m3d.texture[i].h; u32 ch = m3d.texture[i].f;
 		u8[] tex_data = m3d.texture[i].d[0 .. w * h * ch];
 
+		Logf("w: %s h: %s ch: %s", w, h, ch);
+
 		const(char)[] tex_name = m3d.texture[i].name[0 .. strlen(m3d.texture[i].name)];
 		CreateImageView(&rd.vk, &model.textures[i], w, h, ch, tex_data);
 

src/gears/vulkan.d

@@ -340,7 +340,7 @@ InitConversionPipeline(Vulkan* vk)
 		u32 channels = 1;
 
 		CompPipelineInfo conv_info = {
-			shader: "shaders/convert.comp",
+			shader: "shaders/convert.comp.spv",
 			layout: &vk.conv_pipeline_layout,
 			spec: {
 				data: &channels,
@@ -721,6 +721,8 @@ CreateImageView(Vulkan* vk, ImageView* view, u32 w, u32 h, u32 ch, u8[] data)
 {
 	CreateImageView(vk, view, w, h);
 
+	Logf("w: %s h: %s ch: %s", w, h, ch);
+
 	if (ch == 4)
 	{
 		assert(Transfer(vk, view, data, w, h), "CreateImageView failure: Image Transfer error");
@@ -1885,8 +1887,9 @@ VkCheck(string message, VkResult result)
 	{
 		success = false;
 		char[512] buf;
+		buf[] = '\0';
 		buf.sformat("%s: %s", message, VkResultStr(result));
-		Log(buf.ptr);
+		Logf("%r", buf);
 	}
 
 	return success;

src/generated/assets_codegen.d

@@ -1,15 +1,5 @@
 import aliases;
 
-enum AssetType : u32
-{
-	None,
-	Model,
-	Shader,
-	Texture,
-}
-
-alias AT = AssetType;
-
 static immutable string[] MODEL_FILES = [
 	"models/cube.m3d",
 	"models/test_char.m3d",
@@ -28,6 +18,24 @@ static immutable u64[] MODEL_HASHES = [
 	4559395153940738542,
 ];
 
+static immutable string[] STYLIZEDNATURE_FILES = [
+	"StylizedNature/Textures",
+	"StylizedNature/Nature",
+	"StylizedNature/Ruins",
+];
+
+static immutable string[] STYLIZEDNATURE_NAMES = [
+	"StylizedNature/Textures",
+	"StylizedNature/Nature",
+	"StylizedNature/Ruins",
+];
+
+static immutable u64[] STYLIZEDNATURE_HASHES = [
+	13989528115033644773,
+	6938651194392675090,
+	4760645683512501027,
+];
+
 static immutable string[] SHADER_FILES = [
 	"shaders/gui.frag.spv",
 	"shaders/triangle.vert.spv",

src/packer/packer.d

@@ -1,21 +1,37 @@
 import aliases;
 import includes;
 import std.stdio;
+import std.string;
 import std.file;
 import util;
+import std.path;
 import assets;
 import assets_codegen;
 import std.traits;
 import std.algorithm.comparison;
 
+AssetType[string] Lookup = [
+	".m3d": AT.ModelM3D,
+	".png": AT.Texture,
+	".jpg": AT.Texture,
+	".spv": AT.Shader,
+];
+
+u64 ASSET_COUNT = 0;
+string[] FILE_NAMES = [];
+
+/**************************************************
+ ****** UPDATE FILE_VERSION AFTER CHANGES !! ******
+ **************************************************/
+
 void main()
 {
+	Log("running");
 	if (isDir("build"))
 	{
 		chdir("build");
 	}
 
-
 	PackFile();
 	TestFile();
 }
@@ -33,26 +49,41 @@ PackFile()
 
 	chdir("../assets");
 
-	FileHeader h = InitHeader();
+	foreach(string file; dirEntries(".", SpanMode.depth))
+	{
+		if (isDir(file)) continue;
+
+		FILE_NAMES ~= file;
+		ASSET_COUNT += 1;
+	}
+
+	FileHeader h = InitHeader(ASSET_COUNT);
 	ap.rawWrite([h]);
 
-	u64 offset = FileHeader.sizeof + (AssetInfo.sizeof * h.asset_count);
+	u64 offset = FileHeader.sizeof + (AssetInfo.sizeof * ASSET_COUNT);
 	AssetInfo[] asset_info;
-	foreach(type; [ EnumMembers!AssetType ])
+	foreach(file; FILE_NAMES)
 	{
-		auto files = GetAssetFiles(type);
+		AssetType type = AT.None;
-		auto names = GetAssetNames(type);
+		foreach(extension, t; Lookup)
-		auto hashes = GetAssetHashes(type);
+		{
+			if (file.endsWith(extension))
+			{
+				type = t;
+				break;
+			}
+		}
+
+		assert(type != AT.None, "Asset Type is none, offending file " ~ file);
 
-		foreach(i, file; files)
-		{
 		auto f = File(file, "rb");
 		scope(exit) f.close();
 
 		u64 length = cast(u64)f.size();
 
+		string base_name = chompPrefix(file, "./");
 		AssetInfo info = {
-				hash: hashes[i],
+			hash: Hash(base_name),
 			offset: offset,
 			length: length,
 			type: type,
@@ -66,19 +97,8 @@ PackFile()
 
 		offset += length;
 
-			if (type == AT.Texture)
-			{
-				info.texture = GetTexMeta(data);
-			}
-			
-			if (type == AT.Model)
-			{
-				info.model = GetModelMeta(data);
-			}
-
 		asset_info ~= info;
 	}
-	}
 
 	ap.seek(FileHeader.sizeof);
 	ap.rawWrite(asset_info);
@@ -124,22 +144,16 @@ TestFile()
 	}
 
 	FileHeader file_header = ap.rawRead(new FileHeader[1])[0];
-	FileHeader test_header = InitHeader();
+	FileHeader test_header = InitHeader(ASSET_COUNT);
 	assert(file_header == test_header, "TestFile failure: Header is incorrect");
 
-	AssetInfo[] file_info = ap.rawRead(new AssetInfo[file_header.asset_count]);
+	AssetInfo[] file_info = ap.rawRead(new AssetInfo[ASSET_COUNT]);
 	assert(file_info.length == file_header.asset_count, "TestFile failure: Incorrect AssetInfo length returned");
 
 	chdir("../assets");
 
 	u64 asset_index = 0;
-	foreach(type; [ EnumMembers!AssetType ])
+	foreach(i, file; FILE_NAMES)
-	{
-		auto files = GetAssetFiles(type);
-		auto names = GetAssetNames(type);
-		auto hashes = GetAssetHashes(type);
-	
-		foreach(i, file; files)
 	{
 		scope(exit) asset_index += 1;
 
@@ -149,58 +163,12 @@ TestFile()
 		u8[] data = asset.rawRead(new u8[asset.size()]);
 		assert(data.length == info.length, "TestFile failure: File length read is incorrect");
 
-			assert(hashes[i] == info.hash, "TestFile failure: File hash is incorrect");
+		string base_name = chompPrefix(file, "./");
+		assert(Hash(base_name) == info.hash, "TestFile failure: File hash is incorrect");
 
 		ap.seek(info.offset);
 		u8[] pack_data = ap.rawRead(new u8[info.length]);
 		assert(equal!((a, b) => a == b)(data[], pack_data[]), "TestFile failure: Asset data does not match file data");
-
-			if (type == AT.Texture)
-			{
-				assert(info.texture == GetTexMeta(pack_data), "TestFile failure: Texture meta is incorrect");
-			}
-
-			if (type == AT.Model)
-			{
-				assert(info.model == GetModelMeta(pack_data), "TestFile failure: Model meta is incorrect");
-			}
-		}
-	}
-}
-
-immutable(string[])
-GetAssetFiles(AssetType type)
-{
-	switch (type)
-	{
-	case AT.Model: return MODEL_FILES;
-	case AT.Shader: return SHADER_FILES;
-	case AT.Texture: return TEXTURE_FILES;
-	default: return [];
-	}
-}
-
-immutable(string[])
-GetAssetNames(AssetType type)
-{
-	switch (type)
-	{
-	case AT.Model: return MODEL_NAMES;
-	case AT.Shader: return SHADER_NAMES;
-	case AT.Texture: return TEXTURE_NAMES;
-	default: return [];
-	}
-}
-
-immutable(u64[])
-GetAssetHashes(AssetType type)
-{
-	switch (type)
-	{
-	case AT.Model: return MODEL_HASHES;
-	case AT.Shader: return SHADER_HASHES;
-	case AT.Texture: return TEXTURE_HASHES;
-	default: return [];
 	}
 }
 
@@ -212,12 +180,12 @@ MagicValue(string str)
 }
 
 static FileHeader
-InitHeader()
+InitHeader(u64 asset_count)
 {
 	FileHeader header = {
 		magic: MagicValue("steg"),
 		file_version: FILE_VERSION,
-		asset_count: cast(u64)(MODEL_FILES.length + SHADER_FILES.length + TEXTURE_FILES.length),
+		asset_count: asset_count,
 		asset_info_offset: FileHeader.sizeof,
 	};
 

src/shaders/pbr.vert.glsl

@@ -22,7 +22,7 @@ mat4 y_matrix = mat4(
 
 void main()
 {
-	gl_Position = in_pos * y_matrix * G.world_matrix;
+	gl_Position = in_pos * y_matrix;
 
 	if (Materials[nonuniformEXT(PC.mat_id)].albedo_has_texture)
 	{

src/shared/assets.d

@@ -5,28 +5,25 @@ import util;
 import std.exception;
 import alloc;
 
-version(codegen) 
-{
-	const u64 ASSET_COUNT = 0;
-
-	enum AssetType { None };
-}
-else 
-{ 
-	import assets_codegen;
-
-	const u64 ASSET_COUNT = MODEL_FILES.length + SHADER_FILES.length + TEXTURE_FILES.length;
-}
-
 File Asset_File;
 
 FileHeader Asset_Header;
 
-AssetInfo[ASSET_COUNT] Asset_Info;
+AssetInfo[] Asset_Info;
 
-u8[][ASSET_COUNT] Asset_Data;
+u8[][] Asset_Data;
 
-const u32 FILE_VERSION = 1;
+const u32 FILE_VERSION = 2;
+
+enum AssetType : u32
+{
+	None,
+	ModelM3D,
+	Shader,
+	Texture,
+}
+
+alias AT = AssetType;
 
 struct FileHeader
 {
@@ -56,11 +53,6 @@ struct TexMeta
 
 struct AssetInfo
 {
-	union
-	{
-		ModelMeta model;
-		TexMeta texture;
-	};
 	u64 hash;
 	u64 offset;
 	u64 length;
@@ -84,7 +76,6 @@ OpenAssetPack()
 		}
 		catch (ErrnoException e)
 		{
-
 			Logf("OpenAssetPack failure: Unable to open file %s", file_path);
 			assert(false, "Unable to open asset pack file");
 		}
@@ -95,6 +86,9 @@ OpenAssetPack()
 
 		Asset_Header = header_arr[0];
 
+		Asset_Info = AllocArray!(AssetInfo)(Asset_Header.asset_count);
+		Asset_Data = AllocArray!(u8[])(Asset_Header.asset_count);
+
 		assert(Asset_Header.file_version == FILE_VERSION, "OpenAssetPack failure: file version incorrect");
 
 		Asset_File.seek(Asset_Header.asset_info_offset);
@@ -112,6 +106,28 @@ CheckAssetPack()
 	}
 }
 
+AssetInfo
+GetAssetInfo(string name)
+{
+	CheckAssetPack();
+
+	u64 hash = Hash(name);
+
+	AssetInfo asset_info;
+	foreach(i, info; Asset_Info)
+	{
+		if (info.hash == hash)
+		{
+			asset_info = info;
+			break;
+		}
+	}
+
+	assert(asset_info.hash != 0, "GetAssetInfo failure: unable to find matching asset");
+
+	return asset_info;
+}
+
 u8[]
 LoadAssetData(Arena* arena, string name)
 {

src/shared/includes.c

@@ -29,3 +29,4 @@
 #define M3D_IMPLEMENTATION
 
 #include "../../external/m3d/m3d.h"
+