diff --git a/assets.d b/assets.d index b8fe1bd..f81f88c 100644 --- a/assets.d +++ b/assets.d @@ -17,47 +17,33 @@ AssetInfo[] Asset_Info; u8[][] Asset_Data; +__gshared u8[] DEFAULT_TEXTURE = GenerateDefaultTexture(32, 32); + const u32 FILE_VERSION = 3; const u32 MODEL_VERSION = 2; +// TODO: +// Alignment? + /************************* ****** FILE PACKER ****** *************************/ struct FileHeader +{ + u32 magic; + u32 file_version; + u64 asset_count; + u64 asset_info_offset; + DirHeader base_dir; +} + +struct DirHeader { u32 magic; - u32 file_version; + u64 hash; + u64 dir_count; u64 asset_count; - u64 asset_info_offset; -} - -struct ModelHeader -{ - u32 magic; - u32 model_version; - u64 vertex_count; - u64 vertex_offset; - u64 index_count; - u64 index_offset; - u64 material_count; - u64 material_offset; -} - -struct PackedMaterial -{ - StringID name; - Vec3[MatColor.max] colors; - StringHeader[MatMap.max] maps; -} - -struct PackedModel -{ - StringID name; - PackedMaterial[] mats; - Vertex[] vtx; - u32[] idx; - string[] strs; } /************************* @@ -100,11 +86,11 @@ struct MeshPart u64 length; } -struct ModelData +struct Model { - string name; - Mesh[] meshes; - MaterialData[] mats; + string name; + Mesh[] meshes; + Material[] mats; } struct MaterialMap @@ -121,22 +107,18 @@ struct Material Vec4 params; } -struct MaterialData -{ - string name; - - Vec3[MatColor.max] colors; - string[MatMap.max] maps; - f32[MatFloat.max] props = 0.0; - IllumModel illum; -} - struct StringHeader { u32 id; u32 length; } +struct AssetData +{ + AssetType type; + u8[] data; +} + /************************ *** ASSET STRUCTURES *** ************************/ @@ -247,6 +229,24 @@ debug return f.rawRead(mem); } + extern (C) cgltf_result + GLTFLoadCallback(cgltf_memory_options* memory_opts, cgltf_file_options* file_opts, const(char)* path, cgltf_size* size, void** data) + { + u8[] file_data = OpenFile(ConvToStr(path[0 .. strlen(path)])); + + if(file_data == null) return cgltf_result_io_error; + + *size = cast(cgltf_size)file_data.length; + *data = Alloc!(u8)(file_data).ptr; + + return cgltf_result_success; + } + + extern (C) void + GLTFFreeCallback(cgltf_memory_options* memory_opts, cgltf_file_options* file_opts, void* data, cgltf_size size) + { + Free(data[0 .. size]); + } } else { @@ -384,18 +384,85 @@ else } } -PackedModel -Convert(Arena* arena, ModelData* md) +static u8[] +GenerateDefaultTexture(u64 x, u64 y) { - PackedModel packed = { - name: 0, - strs: Alloc!(string)(arena, 1+md.mats.length), - }; + u64 tex_size = x*y*4; + u8[] placeholder_tex = new u8[tex_size]; - packed.strs[0] = Alloc(arena, md.name); - + u8[4] magenta = [255, 0, 255, 255]; + u8[4] black = [0, 0, 0, 255]; + u64 half = tex_size/2; + for(u64 i = 0; i < tex_size; i += 32) + { + bool swap = i <= half; + for(u64 j = 0; j < 16; j += 4) + { + placeholder_tex[i+j .. i+j+4] = !swap ? magenta[0 .. $] : black[0 .. $]; + placeholder_tex[i+j+16 .. i+j+16+4] = !swap ? black[0 .. $] : magenta[0 .. $]; + } + } - return packed; + return placeholder_tex; } +Model +LoadGLTF(Arena* arena, string file_name) +{ + Model model; + + u8[] file_data; // = OpenFile(file_name); + + cgltf_options opts; + cgltf_data* data; + + opts.file.read = &GLTFLoadCallback; + opts.file.release = &GLTFFreeCallback; + + cgltf_result result = cgltf_parse(&opts, file_data.ptr, file_data.length, &data); + + if(result == cgltf_result_success) + { + result = cgltf_load_buffers(&opts, data, file_name.ptr); + if(result != cgltf_result_success) + { + Logf("%s Failure: Unable to load buffers", __FUNCTION__); + } + + u64 primitive_count; + + for(u64 i = 0; i < data.nodes_count; i += 1) + { + cgltf_node* node = &data.nodes[i]; + + if(node.mesh == null) continue; + + for(u64 j = 0; j < node.mesh.primitives_count; j += 1) + { + if(node.mesh.primitives[j].type == cgltf_primitive_type_triangles) + { + primitive_count += 1; + } + } + } + + model.meshes = Alloc!(Mesh)(arena, primitive_count); + model.mats = Alloc!(Material)(arena, data.materials_count+1); + + // Make and load default material into model.mats[0] + + string file_path = GetFilePath(file_name); + for(u64 i = 0; i < data.materials_count; i += 1) + { + // model.mats[i+1] = default_material; + + if(data.materials[i].has_pbr_metallic_roughness) + { + + } + } + } + + return model; +} diff --git a/packer.d b/packer.d index 77e3bca..2a66968 100644 --- a/packer.d +++ b/packer.d @@ -85,15 +85,14 @@ union MeshIdx u32[4] arr; } -struct Model +struct DirEntry { - Vertex[] v; - u32[] idx; - MaterialData[] mats; + string dir; + string[] files; + DirEntry[] sub_dirs; } u64 g_asset_count = 0; -string[] g_file_names = []; Texture[] g_model_textures = []; /************************************************** @@ -102,13 +101,12 @@ Texture[] g_model_textures = []; void main(string[] argv) { - bool pack = false; - bool out_dir = false; - string font_file; + string assets_dir = null; + string asset_pack_path = null; for(u64 i = 0; i < argv.length; i += 1) { - if(argv[i] == "-dir" && i+1 < argv.length) + if(argv[i] == "-out" && i+1 < argv.length) { if(!exists(argv[i+1])) { @@ -120,20 +118,28 @@ void main(string[] argv) assert(false, "Out directory is not a directory"); } - chdir(argv[i+1]); - out_dir = true; + asset_pack_path = argv[i+1]; + i += 1; continue; } - if(argv[i] == "-pack") + if(argv[i] == "-assets") { - pack = true; + if(!exists(argv[i+1])) + { + assert(false, "Assets directory not found"); + } + + assets_dir = argv[i+1]; + + i += 1; + continue; } } - if(!out_dir) + if(asset_pack_path != null) { if(!Cwd!(string)().endsWith("build")) { @@ -144,16 +150,157 @@ void main(string[] argv) if(exists("build") && isDir("build")) { - chdir("build"); + assets_pack_path = "./build/assets.sgp"; } else assert(false, "Unable to make default build directory"); } } + if(assets_dir == null) + { + assert(false, "No assets directory provided"); + } + if(pack) { - PackFile(); - debug TestFile(); + PackFile(assets_pack_path, assets_dir); + debug TestFile(assets_pack_path, assets_dir); + } +} + +void +ScanDir(DirEntry* entry, string dir) +{ + string cwd = Cwd!(string)(); + + + foreach(string e; dirEntries(dir, SpanMode.shallow)) + { + if(e == ".." || e == ".") continue; + + if(isDir(e)) + { + entry.sub_dirs ~= DirEntry(e, []. []); + } + + if(isFile(e)) + { + entry.files ~= e; + } + } + + for(u64 i = 0; i < entry.sub_dirs.length; i += 1) + { + ScanDir(&entry.sub_dirs[i], entry.sub_dirs[i].dir); + } +} + +void +PackFile(string file_path, string assets_dir) +{ + string cwd = Cwd!(string)(); + + File ap = File(file_path, "wb"); + + ChDir(assets_dir); + + DirEntry base_dir = [ + { + dir: "", + files: [], + sub_dirs: [], + }, + ]; + + ScanDir(&base_dir, "."); + + FileHeader h = InitHeader(g_asset_count); + ap.rawWrite([h]); + + u64 offset = FileHeader.sizeof + (AssetInfo.sizeof * g_asset_count); + AssetInfo[] asset_info; + foreach(file; g_file_names) + { + AssetType type = AT.None; + foreach(extension, t; Lookup) + { + if (file.endsWith(extension)) + { + type = t; + break; + } + } + + assert(type != AT.None, "Asset Type is none, offending file " ~ file); + + auto f = File(file, "rb"); + + u64 length = cast(u64)f.size(); + + string base_name = chompPrefix(file, "./"); + AssetInfo info = { + hash: Hash(base_name), + offset: offset, + length: length, + type: type, + }; + + auto data = f.rawRead(new u8[length]); + + assert(length == data.length, "rawRead failure: data length returned doesn't match"); + + ap.seek(offset); + ap.rawWrite(data); + + offset += length; + + asset_info ~= info; + + f.close(); + } + + ap.seek(FileHeader.sizeof); + ap.rawWrite(asset_info); + ap.flush(); + ap.close(); + + ChDir(cwd); +} + +void +TestFile(string file_path, string assets_dir) +{ + File ap = File(file_path, "rb"); + scope(exit) + { + ap.flush(); + ap.close(); + } + + FileHeader file_header = ap.rawRead(new FileHeader[1])[0]; + FileHeader test_header = InitHeader(g_asset_count); + assert(file_header == test_header, "TestFile failure: Header is incorrect"); + + AssetInfo[] file_info = ap.rawRead(new AssetInfo[g_asset_count]); + assert(file_info.length == file_header.asset_count, "TestFile failure: Incorrect AssetInfo length returned"); + + u64 asset_index = 0; + foreach(i, file; g_file_names) + { + scope(exit) asset_index += 1; + + AssetInfo* info = file_info.ptr + asset_index; + File asset = File(file, "rb"); + + u8[] data = asset.rawRead(new u8[asset.size()]); + assert(data.length == info.length, "TestFile failure: File length read is incorrect"); + + string base_name = chompPrefix(file, "./"); + assert(Hash(base_name) == info.hash, "TestFile failure: File hash is incorrect"); + + ap.seek(info.offset); + u8[] pack_data = ap.rawRead(new u8[info.length]); + assert(equal!((a, b) => a == b)(data[], pack_data[]), "TestFile failure: Asset data does not match file data"); } } @@ -276,115 +423,6 @@ GetMatColor(string str) return GetMatColor(GetMatProp(str)); } -void -PackFile() -{ - File ap = File("./assets.sgp", "wb"); - scope(exit) - { - ap.flush(); - ap.close(); - chdir("../build"); - } - - chdir("../assets"); - - foreach(string file; dirEntries(".", SpanMode.depth)) - { - if (isDir(file)) continue; - - g_file_names ~= file; - g_asset_count += 1; - } - - FileHeader h = InitHeader(g_asset_count); - ap.rawWrite([h]); - - u64 offset = FileHeader.sizeof + (AssetInfo.sizeof * g_asset_count); - AssetInfo[] asset_info; - foreach(file; g_file_names) - { - AssetType type = AT.None; - foreach(extension, t; Lookup) - { - if (file.endsWith(extension)) - { - type = t; - break; - } - } - - assert(type != AT.None, "Asset Type is none, offending file " ~ file); - - auto f = File(file, "rb"); - - u64 length = cast(u64)f.size(); - - string base_name = chompPrefix(file, "./"); - AssetInfo info = { - hash: Hash(base_name), - offset: offset, - length: length, - type: type, - }; - - auto data = f.rawRead(new u8[length]); - - assert(length == data.length, "rawRead failure: data length returned doesn't match"); - - ap.seek(offset); - ap.rawWrite(data); - - offset += length; - - asset_info ~= info; - - f.close(); - } - - ap.seek(FileHeader.sizeof); - ap.rawWrite(asset_info); -} - -void -TestFile() -{ - File ap = File("assets.sgp", "rb"); - scope(exit) - { - ap.flush(); - ap.close(); - chdir("../build"); - } - - FileHeader file_header = ap.rawRead(new FileHeader[1])[0]; - FileHeader test_header = InitHeader(g_asset_count); - assert(file_header == test_header, "TestFile failure: Header is incorrect"); - - AssetInfo[] file_info = ap.rawRead(new AssetInfo[g_asset_count]); - assert(file_info.length == file_header.asset_count, "TestFile failure: Incorrect AssetInfo length returned"); - - chdir("../assets"); - - u64 asset_index = 0; - foreach(i, file; g_file_names) - { - scope(exit) asset_index += 1; - - AssetInfo* info = file_info.ptr + asset_index; - File asset = File(file, "rb"); - - u8[] data = asset.rawRead(new u8[asset.size()]); - assert(data.length == info.length, "TestFile failure: File length read is incorrect"); - - string base_name = chompPrefix(file, "./"); - assert(Hash(base_name) == info.hash, "TestFile failure: File hash is incorrect"); - - ap.seek(info.offset); - u8[] pack_data = ap.rawRead(new u8[info.length]); - assert(equal!((a, b) => a == b)(data[], pack_data[]), "TestFile failure: Asset data does not match file data"); - } -} static u32 MagicValue(string str) @@ -406,17 +444,6 @@ InitHeader(u64 asset_count) return header; } -static ModelHeader -InitModelHeader() -{ - ModelHeader header = { - magic: MagicValue("stgm"), - model_version: MODEL_VERSION, - }; - - return header; -} - string[][] TokenizeLines(u8[] data) { @@ -471,13 +498,12 @@ OpenFile(string file_name) return data; } +/* Model ConvertObj(string file_name) { - /* - TODO: - - Deduplicate vertices - */ + // TODO: + // - Deduplicate vertices u8[] data = OpenFile(file_name); @@ -687,109 +713,7 @@ ConvertObj(string file_name) return model; } - -extern (C) cgltf_result -GLTFLoadCallback(cgltf_memory_options* memory_opts, cgltf_file_options* file_opts, const(char)* path, cgltf_size* size, void** data) -{ - u8[] file_data = OpenFile(ConvToStr(path[0 .. strlen(path)])); - - if(file_data == null) return cgltf_result_io_error; - - *size = cast(cgltf_size)file_data.length; - *data = Alloc!(u8)(file_data).ptr; - - return cgltf_result_success; -} - -extern (C) void -GLTFFreeCallback(cgltf_memory_options* memory_opts, cgltf_file_options* file_opts, void* data, cgltf_size size) -{ - Free(data[0 .. size]); -} - -ModelData -LoadGLTF(string file_name) -{ - ModelData model; - - u8[] file_data = OpenFile(file_name); - - cgltf_options opts; - cgltf_data* data; - - opts.file.read = &GLTFLoadCallback; - opts.file.release = &GLTFFreeCallback; - - cgltf_result result = cgltf_parse(&opts, file_data.ptr, file_data.length, &data); - - if(result == cgltf_result_success) - { - result = cgltf_load_buffers(&opts, data, file_name.ptr); - if(result != cgltf_result_success) - { - Logf("%s Failure: Unable to load buffers", __FUNCTION__); - } - - u64 primitive_count; - - for(u64 i = 0; i < data.nodes_count; i += 1) - { - cgltf_node* node = &data.nodes[i]; - - if(node.mesh == null) continue; - - for(u64 j = 0; j < mesh.primitives_count; j += 1) - { - if(mesh.primitives[j].type == cgltf_primtive_type_triangles) - { - primitive_count += 1; - } - } - } - - model.meshes = new Mesh[primitive_count]; - model.mats = new MaterialData[data.materials_count+1]; - - // Make and load default material into model.mats[0] - - string file_path = GetFilePath(file_name); - for(u64 i = 0; i < data.materials_count; i += 1) - { - // model.mats[i+1] = default_material; - - if(data.materials[i].has_pbr_metallic_roughness) - { - - } - } - } -} - -string -GetFilePath(string file_name) -{ - string result = file_name; - - for(u64 i = file_name.length-1; i64(i) >= 0; i -= 1) - { - version(Windows) - { - char ch = '\\'; - } - else - { - char ch = '/'; - } - - if(file_name[i] == ch) - { - result = file_name[0 .. i+1]; - break; - } - } - - return result; -} +*/ pragma(inline) f32 ToF32(string str) @@ -818,7 +742,7 @@ CheckWhiteSpace(u8 ch) unittest { { // Obj test - Model model = ConvertObj("./test/sponza.obj"); + //Model model = ConvertObj("./test/sponza.obj"); } { diff --git a/util.d b/util.d index 28b55f4..0902682 100644 --- a/util.d +++ b/util.d @@ -154,6 +154,32 @@ ConvertColor(Vec4 *dst, u32 src) } } +string +GetFilePath(string file_name) +{ + string result = file_name; + + for(u64 i = file_name.length-1; i64(i) >= 0; i -= 1) + { + version(Windows) + { + char ch = '\\'; + } + else + { + char ch = '/'; + } + + if(file_name[i] == ch) + { + result = file_name[0 .. i+1]; + break; + } + } + + return result; +} + pragma(inline) void Convert(Vec4* dst, u32 src) {