complete asset packer, add stb_image, m3d

build-vma.sh

@@ -34,3 +34,27 @@ if ! [ -f build/libxxhash.a ]; then
 	ar rcs $lib $obj
 	rm $obj
 fi
+
+# STB_IMAGE
+src="external/stb/stb_image.c"
+flags="-std=c99 -Wno-everything -Iexternal/stb -c -static"
+obj="build/stb_image.o"
+lib="build/libstb_image.a"
+
+if ! [ -f build/libstb_image.a ]; then
+	$c_compiler $flags $src $out $obj
+	ar rcs $lib $obj
+	rm $obj
+fi
+
+# M3D
+src="external/m3d/m3d.c"
+flags="-std=c99 -Wno-everything -Iexternal/m3d -c -static"
+obj="build/m3d.o"
+lib="build/libm3d.a"
+
+if ! [ -f build/libm3d.a ]; then
+	$c_compiler $flags $src $out $obj
+	ar rcs $lib $obj
+	rm $obj
+fi

dub.json

@@ -7,10 +7,10 @@
 			"targetType": "executable",
 			"targetName": "Gears",
 			"targetPath": "build",
-			"sourceFiles-linux": ["build/libvma.a", "build/libxxhash.a"],
+			"sourceFiles-linux": ["build/libvma.a", "build/libxxhash.a", "build/libstb_image.a"],
 			"sourceFiles-windows": [],
-			"importPaths": ["src/gears", "src/shared"],
+			"importPaths": ["src/gears", "src/shared", "src/generated"],
-			"sourcePaths": ["src/gears", "src/shared"],
+			"sourcePaths": ["src/gears", "src/shared", "src/generated"],
 			"libs-linux": ["xcb", "X11", "X11-xcb", "vulkan", "stdc++"],
 			"libs-windows": [],
 			"preGenerateCommands-linux": ["./build-vma.sh"],
@@ -21,9 +21,9 @@
 			"targetType": "executable",
 			"targetPath": "build",
 			"targetName": "Packer",
-			"importPaths": ["src/packer", "src/shared"],
+			"importPaths": ["src/packer", "src/shared", "src/generated"],
-			"sourcePaths": ["src/packer", "src/shared"],
+			"sourcePaths": ["src/packer", "src/shared", "src/generated"],
-			"sourceFiles-linux": ["build/libxxhash.a"],
+			"sourceFiles-linux": ["build/libxxhash.a", "build/libstb_image.a", "build/libm3d.a"],
 			"preGenerateCommands-linux": ["./build-vma.sh"],
 			"preGenerateCommands-windows": [],
 		},
@@ -34,7 +34,7 @@
 			"targetName": "Codegen",
 			"importPaths": ["src/codegen", "src/shared"],
 			"sourcePaths": ["src/codegen", "src/shared"],
-			"sourceFiles-linux": ["build/libxxhash.a"],
+			"sourceFiles-linux": ["build/libxxhash.a", "build/libstb_image.a"],
 			"preGenerateCommands-linux": ["./build-vma.sh"],
 			"preGenerateCommands-windows": [],
 		}

external/m3d/m3d.c

@@ -0,0 +1,3 @@
+#define M3D_IMPLEMENTATION
+
+#include "m3d.h"

external/stb/stb_image.c

@@ -0,0 +1,3 @@
+#define STB_IMAGE_IMPLEMENTATION
+
+#include "stb_image.h"

src/codegen/codegen.d

@@ -4,6 +4,9 @@ import std.stdio;
 import std.file;
 import std.string;
 import std.path;
+import std.uni;
+import std.conv;
+import std.array;
 
 void
 main()
@@ -55,21 +58,50 @@ CreateAssetLookups()
 		file_hashes[dir] = hashes;
 	}
 
-	auto f = File("src/generated/assets.d", "w");
+	auto f = File("src/generated/assets_codegen.d", "w");
+	scope(exit) f.close();
+
+	string asset_str = "import aliases;\n\n";
+
+	asset_str ~= "enum AssetType : u32\n{\n\tNone,\n";
+	foreach(i, dir; dirs)
+	{
+		auto graphemes = dir.byGrapheme.array;
+		string upper = [graphemes[0]].byCodePoint.text.toUpper;
+		graphemes[0] = upper.decodeGrapheme;
+
+		string asset_type = graphemes.byCodePoint.text.stripRight("s");
+		asset_str ~= format("\t%s,\n", asset_type);
+	}
+	asset_str ~= "}\n\nalias AT = AssetType;\n\n";
 
 	foreach(dir; dirs)
 	{
 		string asset_type = stripRight(dir, "s").toUpper();
 
-		string file_array = format("static immutable string[] %s_FILES = [\n", asset_type);
+		asset_str ~= format("static immutable string[] %s_FILES = [\n", asset_type);
 		foreach(file; file_names[dir])
 		{
-			file_array ~= format("\t\"%s\",\n", file);
+			asset_str ~= format("\t\"%s\",\n", file);
 		}
-		file_array ~= "];\n\n";
+		asset_str ~= "];\n\n";
+
+
+		asset_str ~= format("static immutable string[] %s_NAMES = [\n", asset_type);
+		foreach(name; base_names[dir])
+		{
+			asset_str ~= format("\t\"%s\",\n", name);
+		}
+		asset_str ~= "];\n\n";
+
+		asset_str ~= format("static immutable u64[] %s_HASHES = [\n", asset_type);
+		foreach(hash; file_hashes[dir])
+		{
+			asset_str ~= format("\t%d,\n", hash);	
+		}
+		asset_str ~= "];\n\n";
 
-		f.write(file_array);
 	}
 
-	f.close();
+	f.write(asset_str);
 }

src/generated/assets_codegen.d

@@ -0,0 +1,84 @@
+import aliases;
+
+enum AssetType : u32
+{
+	None,
+	Model,
+	Shader,
+	Texture,
+}
+
+alias AT = AssetType;
+
+static immutable string[] MODEL_FILES = [
+	"models/test_char.m3d",
+	"models/yoda.m3d",
+];
+
+static immutable string[] MODEL_NAMES = [
+	"models/test_char",
+	"models/yoda",
+];
+
+static immutable u64[] MODEL_HASHES = [
+	13826959199295087925,
+	4559395153940738542,
+];
+
+static immutable string[] SHADER_FILES = [
+	"shaders/gui.frag.spv",
+	"shaders/pbr.frag.spv",
+	"shaders/gui.vert.spv",
+	"shaders/quad.frag.spv",
+	"shaders/quad.vert.spv",
+	"shaders/pbr.vert.spv",
+];
+
+static immutable string[] SHADER_NAMES = [
+	"shaders/gui.frag",
+	"shaders/pbr.frag",
+	"shaders/gui.vert",
+	"shaders/quad.frag",
+	"shaders/quad.vert",
+	"shaders/pbr.vert",
+];
+
+static immutable u64[] SHADER_HASHES = [
+	15780387719315455808,
+	2230071466542309169,
+	14797956403837654625,
+	8430018914716708078,
+	14432191255225961360,
+	8518761701216801634,
+];
+
+static immutable string[] TEXTURE_FILES = [
+	"textures/ham_smoke.png",
+	"textures/cheesoid.png",
+	"textures/hog.jpg",
+	"textures/patamon.png",
+	"textures/pattermon.png",
+	"textures/hamster.png",
+	"textures/purplemon.png",
+];
+
+static immutable string[] TEXTURE_NAMES = [
+	"textures/ham_smoke",
+	"textures/cheesoid",
+	"textures/hog",
+	"textures/patamon",
+	"textures/pattermon",
+	"textures/hamster",
+	"textures/purplemon",
+];
+
+static immutable u64[] TEXTURE_HASHES = [
+	15457434128510259736,
+	12443444479937967236,
+	5538438794723924882,
+	16650761267170532297,
+	11718504567089932798,
+	7627422980398294448,
+	14316598952102237724,
+];
+

src/generated/codegen.d

@@ -1,24 +0,0 @@
-static immutable string[] MODEL_FILES = [
-	"models/test_char.m3d",
-	"models/yoda.m3d",
-];
-
-static immutable string[] SHADER_FILES = [
-	"shaders/gui.frag.spv",
-	"shaders/pbr.frag.spv",
-	"shaders/gui.vert.spv",
-	"shaders/quad.frag.spv",
-	"shaders/quad.vert.spv",
-	"shaders/pbr.vert.spv",
-];
-
-static immutable string[] TEXTURE_FILES = [
-	"textures/ham_smoke.png",
-	"textures/cheesoid.png",
-	"textures/hog.jpg",
-	"textures/patamon.png",
-	"textures/pattermon.png",
-	"textures/hamster.png",
-	"textures/purplemon.png",
-];
-

src/packer/packer.d

@@ -1,9 +1,12 @@
 import aliases;
+import includes;
+import std.stdio;
 import std.file;
 import util;
 import assets;
-
+import assets_codegen;
-const u32 FILE_VERSION = 1;
+import std.traits;
+import std.algorithm.comparison;
 
 void main()
 {
@@ -12,9 +15,190 @@ void main()
 		chdir("build");
 	}
 
-	static foreach(file; AssetFiles)
+
+	PackFile();
+	TestFile();
+}
+
+void
+PackFile()
+{
+	File ap = File("./assets.sgp", "wb");
+	scope(exit) 
 	{
-		Log(file);
+		ap.flush();
+		ap.close();
+		chdir("../build");
+	}
+
+	chdir("../assets");
+
+	FileHeader h = InitHeader();
+	ap.rawWrite([h]);
+
+	u64 offset = FileHeader.sizeof + (AssetInfo.sizeof * h.asset_count);
+	AssetInfo[] asset_info;
+	foreach(type; [ EnumMembers!AssetType ])
+	{
+		auto files = GetAssetFiles(type);
+		auto names = GetAssetNames(type);
+		auto hashes = GetAssetHashes(type);
+	
+		foreach(i, file; files)
+		{
+			auto f = File(file, "rb");
+			scope(exit) f.close();
+
+			u64 length = cast(u64)f.size();
+
+			AssetInfo info = {
+				hash: hashes[i],
+				offset: offset,
+				length: length,
+				type: type,
+			};
+
+			auto data = f.rawRead(new u8[length]);
+			assert(length == data.length, "rawRead failure: data length returned doesn't match");
+
+			ap.seek(offset);
+			ap.rawWrite(data);
+
+			offset += length;
+
+			if (type == AT.Texture)
+			{
+				info.texture = GetTexMeta(data);
+			}
+			
+			if (type == AT.Model)
+			{
+				info.model = GetModelMeta(data);
+			}
+
+			asset_info ~= info;
+		}
+	}
+
+	ap.seek(FileHeader.sizeof);
+	ap.rawWrite(asset_info);
+}
+
+TexMeta
+GetTexMeta(u8[] data)
+{
+	int ch = 4;
+	int width, height, has_ch;
+	auto img = stbi_load_from_memory(data.ptr, cast(int)data.length, &width, &height, &has_ch, ch);
+
+	assert(img != null, "stbi_load_from_image failure: image is NULL");
+	assert(width > 0 && height > 0 && has_ch > 0, "stbi_load_from_image failure: dimensions are invalid");
+
+	stbi_image_free(img);
+
+	return TexMeta(w: width, h: height, ch: has_ch);
+}
+
+ModelMeta
+GetModelMeta(u8[] data)
+{
+	m3d_t* model = m3d_load(data.ptr, null, null, null);
+	assert(model != null, "m3d_load failure: model is null");
+
+	u64 index_count = cast(u64)(model.numface * 3);
+
+	m3d_free(model);
+
+	return ModelMeta(index_count: index_count);
+}
+
+void
+TestFile()
+{
+	File ap = File("assets.sgp", "rb");
+	scope(exit)
+	{
+		ap.flush();
+		ap.close();
+		chdir("../build");
+	}
+
+	FileHeader file_header = ap.rawRead(new FileHeader[1])[0];
+	FileHeader test_header = InitHeader();
+	assert(file_header == test_header, "TestFile failure: Header is incorrect");
+
+	AssetInfo[] file_info = ap.rawRead(new AssetInfo[file_header.asset_count]);
+	assert(file_info.length == file_header.asset_count, "TestFile failure: Incorrect AssetInfo length returned");
+
+	chdir("../assets");
+
+	u64 asset_index = 0;
+	foreach(type; [ EnumMembers!AssetType ])
+	{
+		auto files = GetAssetFiles(type);
+		auto names = GetAssetNames(type);
+		auto hashes = GetAssetHashes(type);
+	
+		foreach(i, file; files)
+		{
+			scope(exit) asset_index += 1;
+
+			AssetInfo* info = file_info.ptr + asset_index;
+			File asset = File(file, "rb");
+
+			u8[] data = asset.rawRead(new u8[asset.size()]);
+			assert(data.length == info.length, "TestFile failure: File length read is incorrect");
+
+			ap.seek(info.offset);
+			u8[] pack_data = ap.rawRead(new u8[info.length]);
+			assert(equal!((a, b) => a == b)(data[], pack_data[]), "TestFile failure: Asset data does not match file data");
+
+			if (type == AT.Texture)
+			{
+				assert(info.texture == GetTexMeta(pack_data), "TestFile failure: Texture meta is incorrect");
+			}
+
+			if (type == AT.Model)
+			{
+				assert(info.model == GetModelMeta(pack_data), "TestFile failure: Model meta is incorrect");
+			}
+		}
+	}
+}
+
+immutable(string[])
+GetAssetFiles(AssetType type)
+{
+	switch (type)
+	{
+	case AT.Model: return MODEL_FILES;
+	case AT.Shader: return SHADER_FILES;
+	case AT.Texture: return TEXTURE_FILES;
+	default: return [];
+	}
+}
+
+immutable(string[])
+GetAssetNames(AssetType type)
+{
+	switch (type)
+	{
+	case AT.Model: return MODEL_NAMES;
+	case AT.Shader: return SHADER_NAMES;
+	case AT.Texture: return TEXTURE_NAMES;
+	default: return [];
+	}
+}
+
+immutable(u64[])
+GetAssetHashes(AssetType type)
+{
+	switch (type)
+	{
+	case AT.Model: return MODEL_HASHES;
+	case AT.Shader: return SHADER_HASHES;
+	case AT.Texture: return TEXTURE_HASHES;
+	default: return [];
 	}
 }
 
@@ -25,12 +209,14 @@ MagicValue(string str)
 	return cast(u32)(cast(u32)(str[0] << 24) | cast(u32)(str[1] << 16) | cast(u32)(str[2] << 8) | cast(u32)(str[3] << 0));
 }
 
-FileHeader
+static FileHeader
 InitHeader()
 {
 	FileHeader header = {
 		magic: MagicValue("steg"),
 		file_version: FILE_VERSION,
+		asset_count: cast(u64)(MODEL_FILES.length + SHADER_FILES.length + TEXTURE_FILES.length),
+		asset_info_offset: FileHeader.sizeof,
 	};
 
 	return header;

src/shared/assets.d

@@ -1,12 +1,15 @@
 import aliases;
 import std.file;
+import assets_codegen;
+
+const u32 FILE_VERSION = 1;
 
 struct FileHeader
 {
 	u32 magic;
 	u32 file_version;
-	u64 asset_counts;
+	u64 asset_count;
-	u64 asset_offset;
+	u64 asset_info_offset;
 }
 
 struct ModelMeta
@@ -21,16 +24,6 @@ struct TexMeta
 	u32 ch;
 }
 
-enum AssetType : u32
-{
-	None,
-	Shader,
-	Texture,
-	Model,
-}
-
-alias AT = AssetType;
-
 struct AssetInfo
 {
 	union
@@ -40,6 +33,6 @@ struct AssetInfo
 	};
 	u64 hash;
 	u64 offset;
-	u64 len;
+	u64 length;
 	AssetType type;
 }

src/shared/includes.c

@@ -22,4 +22,12 @@
 
 #include "../../external/vma/vk_mem_alloc.h"
 
-#include "../../external/xxhash/xxhash.h"
+#include "../../external/xxhash/xxhash.c"
+
+#define STB_IMAGE_IMPLEMENTATION
+
+#include "../../external/stb/stb_image.h"
+
+#define M3D_IMPLEMENTATION
+
+#include "../../external/m3d/m3d.h"