module dlib.packer; import includes; import dlib.util; import dlib.aliases; import dlib.assets; import dlib.platform; import dlib.alloc; import std.stdio; import std.string; import std.file; import std.path; import std.traits; import std.algorithm.comparison; import core.memory; import std.conv; import std.array; AssetType[string] Lookup = [ ".obj": AT.Model, ".png": AT.Texture, ".jpg": AT.Texture, ".spv": AT.Shader, ]; enum MatProp { None, Ambient, Albedo, Specular, SpecularExp, Dissolve, // Transparency 1.0 -> opaque Transparency, // Transparency 0.0 -> opaque Transmission, OpticalDensity, Illumination, AmbientMap, AlbedoMap, SpecularMap, SpecularHighlightMap, AlphaMap, BumpMap, DisplacementMap, Stencil, Roughness, RoughnessMap, Metallic, MetallicMap, Sheen, SheenMap, ClearcoatThickness, ClearcoatRoughness, Emissive, EmissiveMap, Anisotropy, AnisotropyMap, NormalMap, } struct PackedString { StringHeader header; string str; alias header this; } struct Texture { string name; u8[] data; u32 w; u32 h; u32 ch; } union MeshIdx { struct { u32 v, uv, n, t; }; u32[4] arr; } struct Model { Vertex[] v; u32[] idx; MaterialData[] mats; } u64 g_asset_count = 0; string[] g_file_names = []; Texture[] g_model_textures = []; /************************************************** ****** UPDATE FILE_VERSION AFTER CHANGES !! ****** **************************************************/ void main(string[] argv) { bool pack = false; bool out_dir = false; string font_file; for(u64 i = 0; i < argv.length; i += 1) { if(argv[i] == "-dir" && i+1 < argv.length) { if(!exists(argv[i+1])) { mkdir(argv[i+1]); } if(!isDir(argv[i+1])) { assert(false, "Out directory is not a directory"); } chdir(argv[i+1]); out_dir = true; i += 1; continue; } if(argv[i] == "-pack") { pack = true; } } if(!out_dir) { if(!Cwd!(string)().endsWith("build")) { if(!exists("build")) { mkdir("build"); } if(exists("build") && isDir("build")) { chdir("build"); } else assert(false, "Unable to make default build directory"); } } if(pack) { PackFile(); debug TestFile(); } } MatProp GetMatProp(string str) { switch(str) with(MatProp) { // Vec3 case "Ka": return Ambient; case "Kd": return Albedo; case "Ks": return Specular; case "Tf": return Transmission; case "Ke": return Emissive; // Illum case "illum": return Illumination; // string case "map_Ka": return AmbientMap; case "map_Kd": return AlbedoMap; case "map_Ks": return SpecularMap; case "map_Ns": return SpecularHighlightMap; case "map_d": return AlphaMap; case "map_bump": case "bump": return BumpMap; case "map_Pr": return RoughnessMap; case "map_Pm": return MetallicMap; case "map_Ke": return EmissiveMap; case "map_Ps": return SheenMap; case "norm": return NormalMap; case "anisor": return AnisotropyMap; case "disp": return DisplacementMap; case "decal": return Stencil; // f32 case "Ns": return SpecularExp; case "d": return Dissolve; case "Tr": return Transparency; case "Ni": return OpticalDensity; case "Pr": return Roughness; case "Pm": return Metallic; case "Pc": return ClearcoatThickness; case "Pcr": return ClearcoatRoughness; case "aniso": return Anisotropy; case "Ps": return Sheen; default: return None; } } MatColor GetMatColor(MatProp prop) { switch(prop) with(MatProp) { case Ambient: return MatColor.Ambient; case Albedo: return MatColor.Albedo; case Specular: return MatColor.Specular; case Transmission: return MatColor.Transmission; case Emissive: return MatColor.Emissive; default: assert(false, "Unknown MatProp to MatColor conversion"); } } MatFloat GetMatFloat(MatProp prop) { switch(prop) with(MatProp) { case SpecularExp: return MatFloat.SpecularExp; case Transparency: case Dissolve: return MatFloat.Alpha; case OpticalDensity: return MatFloat.OpticalDensity; case Roughness: return MatFloat.Roughness; case Metallic: return MatFloat.Metallic; case ClearcoatThickness: return MatFloat.ClearcoatThickness; case ClearcoatRoughness: return MatFloat.ClearcoatRoughness; case Anisotropy: return MatFloat.Anisotropy; case Sheen: return MatFloat.Sheen; default: assert(false, "Unknown MatProp to MatFloat conversion"); } } MatMap GetMatMap(MatProp prop) { switch(prop) with(MatProp) { case AmbientMap: return MatMap.Ambient; case AlbedoMap: return MatMap.Albedo; case SpecularMap: return MatMap.Specular; case SpecularHighlightMap: return MatMap.SpecularHighlight; case AlphaMap: return MatMap.Alpha; case BumpMap: return MatMap.Bump; case RoughnessMap: return MatMap.Roughness; case MetallicMap: return MatMap.Metallic; case EmissiveMap: return MatMap.Emissive; case SheenMap: return MatMap.Sheen; case NormalMap: return MatMap.Normal; case AnisotropyMap: return MatMap.Anisotropy; case DisplacementMap: return MatMap.Displacement; case Stencil: return MatMap.Ambient; default: assert(false, "Unknown MatProp to MatMap conversion"); } } MatMap GetMatMap(string str) { return GetMatMap(GetMatProp(str)); } MatFloat GetMatFloat(string str) { return GetMatFloat(GetMatProp(str)); } MatColor GetMatColor(string str) { return GetMatColor(GetMatProp(str)); } void PackFile() { File ap = File("./assets.sgp", "wb"); scope(exit) { ap.flush(); ap.close(); chdir("../build"); } chdir("../assets"); foreach(string file; dirEntries(".", SpanMode.depth)) { if (isDir(file)) continue; g_file_names ~= file; g_asset_count += 1; } FileHeader h = InitHeader(g_asset_count); ap.rawWrite([h]); u64 offset = FileHeader.sizeof + (AssetInfo.sizeof * g_asset_count); AssetInfo[] asset_info; foreach(file; g_file_names) { AssetType type = AT.None; foreach(extension, t; Lookup) { if (file.endsWith(extension)) { type = t; break; } } assert(type != AT.None, "Asset Type is none, offending file " ~ file); auto f = File(file, "rb"); u64 length = cast(u64)f.size(); string base_name = chompPrefix(file, "./"); AssetInfo info = { hash: Hash(base_name), offset: offset, length: length, type: type, }; auto data = f.rawRead(new u8[length]); assert(length == data.length, "rawRead failure: data length returned doesn't match"); ap.seek(offset); ap.rawWrite(data); offset += length; asset_info ~= info; f.close(); } ap.seek(FileHeader.sizeof); ap.rawWrite(asset_info); } void TestFile() { File ap = File("assets.sgp", "rb"); scope(exit) { ap.flush(); ap.close(); chdir("../build"); } FileHeader file_header = ap.rawRead(new FileHeader[1])[0]; FileHeader test_header = InitHeader(g_asset_count); assert(file_header == test_header, "TestFile failure: Header is incorrect"); AssetInfo[] file_info = ap.rawRead(new AssetInfo[g_asset_count]); assert(file_info.length == file_header.asset_count, "TestFile failure: Incorrect AssetInfo length returned"); chdir("../assets"); u64 asset_index = 0; foreach(i, file; g_file_names) { scope(exit) asset_index += 1; AssetInfo* info = file_info.ptr + asset_index; File asset = File(file, "rb"); u8[] data = asset.rawRead(new u8[asset.size()]); assert(data.length == info.length, "TestFile failure: File length read is incorrect"); string base_name = chompPrefix(file, "./"); assert(Hash(base_name) == info.hash, "TestFile failure: File hash is incorrect"); ap.seek(info.offset); u8[] pack_data = ap.rawRead(new u8[info.length]); assert(equal!((a, b) => a == b)(data[], pack_data[]), "TestFile failure: Asset data does not match file data"); } } static u32 MagicValue(string str) { assert(str.length == 4, "Magic value must 4 characters"); return cast(u32)(cast(u32)(str[0] << 24) | cast(u32)(str[1] << 16) | cast(u32)(str[2] << 8) | cast(u32)(str[3] << 0)); } static FileHeader InitHeader(u64 asset_count) { FileHeader header = { magic: MagicValue("steg"), file_version: FILE_VERSION, asset_count: asset_count, asset_info_offset: FileHeader.sizeof, }; return header; } static ModelHeader InitModelHeader() { ModelHeader header = { magic: MagicValue("stgm"), model_version: MODEL_VERSION, }; return header; } string[][] TokenizeLines(u8[] data) { string[][] tokens = []; string[] line_tokens = []; u64 start = -1; for(u64 i = 0; i < data.length; i += 1) { if(i64(start) != -1 && CheckWhiteSpace(data[i])) { line_tokens ~= ConvString(data[start .. i]); start = -1; } if(data[i] == '\n') { tokens ~= line_tokens; line_tokens = []; continue; } if(i64(start) == -1 && !CheckWhiteSpace(data[i])) { start = i; continue; } } return tokens; } u8[] OpenFile(string file_name) { File f; u8[] data; try { f = File(file_name, "rb"); data = new u8[f.size()]; f.rawRead(data); } catch(Exception e) { data = null; } f.close(); return data; } Model ConvertObj(string file_name) { /* TODO: - Deduplicate vertices */ u8[] data = OpenFile(file_name); Model model; u64 vcount, uvcount, ncount, fcount, gcount, mcount; string[][] tokens = TokenizeLines(data); for(u64 i = 0; i < tokens.length; i += 1) { if(tokens[i].length == 0) continue; switch(tokens[i][0]) { case "v": vcount += 1; break; case "vt": uvcount += 1; break; case "vn": ncount += 1; break; case "f": fcount += 1; break; case "g": gcount += 1; break; case "usemtl": mcount += 1; break; default: break; } } Vec3[] positions = new Vec3[vcount]; Vec3[] normals = new Vec3[ncount]; Vec2[] uvs = new Vec2[uvcount]; MeshIdx[3][][] idx = []; MaterialData[] mtls = []; vcount = 0; ncount = 0; uvcount = 0; MeshIdx[3][] part_idx = []; for(u64 i = 0; i < tokens.length; i += 1) { if(tokens[i][0] == "#") continue; if(tokens[i][0] == "v") { if(tokens[i].length < 4) assert(false, "OBJ file error, not enough points for case [v]"); positions[vcount++] = Vec3(ToF32(tokens[i][1]), ToF32(tokens[i][2]), ToF32(tokens[i][3])); continue; } if(tokens[i][0] == "vn") { if(tokens[i].length < 4) assert(false, "OBJ file error, not enough points for case [vn]"); normals[ncount++] = Vec3(ToF32(tokens[i][1]), ToF32(tokens[i][2]), ToF32(tokens[i][3])); continue; } if(tokens[i][0] == "vt") { if(tokens[i].length < 3) assert(false, "OBJ file error, not enough points for case [vt]"); uvs[uvcount++] = Vec2(ToF32(tokens[i][1]), ToF32(tokens[i][2])); continue; } if(tokens[i][0] == "f") { u32 sep_count = StrCharCount(tokens[i][1], '/'); if(tokens[i].length == 4) { MeshIdx[3] face; for(u64 j = 1; j < tokens[i].length; j += 1) { string[] parts = tokens[i][j].split('/'); if(sep_count == 0) { face[j-1] = MeshIdx(v: to!u32(parts[0])); } if(sep_count == 1) { face[j-1] = MeshIdx(v: to!u32(parts[0]), uv: to!u32(parts[1])); } if(sep_count == 2) { MeshIdx mesh_idx; foreach(ipart, part; parts) { if(part == "") continue; mesh_idx.arr[ipart] = to!u32(part); } face[j-1] = mesh_idx; } } part_idx ~= face; } else assert(false, "Only triangles or quads supported for mesh face"); continue; } if(tokens[i][0] == "g" && part_idx.length > 0) { idx ~= part_idx; part_idx = []; continue; } if(tokens[i][0] == "mtllib") { u8[] mtl_data = OpenFile(GetFilePath(file_name) ~ tokens[i][1]); MaterialData* mtl = null; string[][] mtl_tokens = TokenizeLines(mtl_data); for(u64 j = 0; j < mtl_tokens.length; j += 1) { if(mtl_tokens[j].length == 0) { if(mtl) { mtls ~= *mtl; mtl = null; } continue; } if(mtl_tokens[j][0] == "newmtl") { mtl = new MaterialData; mtl.name = mtl_tokens[j][1]; continue; } if(!mtl) continue; switch(mtl_tokens[j][0]) { case "Ka", "Kd", "Ks", "Tf", "Ke": { mtl.colors[GetMatColor(mtl_tokens[j][0])] = Vec3(ToF32(mtl_tokens[j][1]), ToF32(mtl_tokens[j][2]), ToF32(mtl_tokens[j][3])); } break; case "Ns", "d", "Tr", "Ni", "Pr", "Pm", "Pc", "Pcr", "aniso", "Ps": { f32 v = ToF32(mtl_tokens[j][1]); if(mtl_tokens[j][1] == "Tr") { v = 1.0 - v; } mtl.props[GetMatFloat(mtl_tokens[j][0])] = v; } break; case "map_Ka", "map_Kd", "map_Ks", "map_Ns", "map_d", "map_bump", "anisor", "bump", "map_Pr", "map_Pm", "map_Ke", "map_Ps", "norm", "disp", "decal": { mtl.maps[GetMatMap(mtl_tokens[j][0])] = mtl_tokens[j][1]; } break; case "illum": { mtl.illum = cast(IllumModel)(to!(u32)(mtl_tokens[j][1])); } break; default: break; } } if(mtl) { mtls ~= *mtl; mtl = null; } } } u64 face_count; foreach(part; idx) { face_count += part.length; } ModelData md = { name: baseName(file_name, ".obj"), meshes: [ { vtx: new Vertex[face_count*3] }, ] }; Logf("%s %s", (Vertex.sizeof * md.vtx.length), (positions.length*Vec3.sizeof + uvs.length*Vec2.sizeof + normals.length*Vec3.sizeof)); u64 vtx_count = 0; foreach(part; idx) { for(u64 i = 0; i < part.length; i += 1) { for(u64 j = 0; j < 3; j += 1) { MeshIdx* mi = part[i].ptr + j; if(mi.v ) md.meshes[0].vtx[vtx_count+j].pos = positions[mi.v-1]; if(mi.n ) md.meshes[0].vtx[vtx_count+j].normal = normals[mi.n-1]; if(mi.uv) md.meshes[0].vtx[vtx_count+j].uv = uvs[mi.uv-1]; } vtx_count += 3; } } return model; } extern (C) cgltf_result GLTFLoadCallback(cgltf_memory_options* memory_opts, cgltf_file_options* file_opts, const(char)* path, cgltf_size* size, void** data) { u8[] file_data = OpenFile(ConvToStr(path[0 .. strlen(path)])); if(file_data == null) return cgltf_result_io_error; *size = cast(cgltf_size)file_data.length; *data = Alloc!(u8)(file_data).ptr; return cgltf_result_success; } extern (C) void GLTFFreeCallback(cgltf_memory_options* memory_opts, cgltf_file_options* file_opts, void* data, cgltf_size size) { Free(data[0 .. size]); } ModelData LoadGLTF(string file_name) { ModelData model; u8[] file_data = OpenFile(file_name); cgltf_options opts; cgltf_data* data; opts.file.read = &GLTFLoadCallback; opts.file.release = &GLTFFreeCallback; cgltf_result result = cgltf_parse(&opts, file_data.ptr, file_data.length, &data); if(result == cgltf_result_success) { result = cgltf_load_buffers(&opts, data, file_name.ptr); if(result != cgltf_result_success) { Logf("%s Failure: Unable to load buffers", __FUNCTION__); } u64 primitive_count; for(u64 i = 0; i < data.nodes_count; i += 1) { cgltf_node* node = &data.nodes[i]; if(node.mesh == null) continue; for(u64 j = 0; j < mesh.primitives_count; j += 1) { if(mesh.primitives[j].type == cgltf_primtive_type_triangles) { primitive_count += 1; } } } model.meshes = new Mesh[primitive_count]; model.mats = new MaterialData[data.materials_count+1]; // Make and load default material into model.mats[0] string file_path = GetFilePath(file_name); for(u64 i = 0; i < data.materials_count; i += 1) { // model.mats[i+1] = default_material; if(data.materials[i].has_pbr_metallic_roughness) { } } } } string GetFilePath(string file_name) { string result = file_name; for(u64 i = file_name.length-1; i64(i) >= 0; i -= 1) { version(Windows) { char ch = '\\'; } else { char ch = '/'; } if(file_name[i] == ch) { result = file_name[0 .. i+1]; break; } } return result; } pragma(inline) f32 ToF32(string str) { return to!(f32)(str); } pragma(inline) string ConvString(u8[] bytes) { return (cast(immutable(char)*)bytes.ptr)[0 .. bytes.length]; } pragma(inline) bool CheckWhiteSpace(u8 ch) { return ch == ' ' || ch == '\t'|| ch == '\n'|| ch == 0x0D|| ch == 0x0A|| ch == 0x0B|| ch == 0x0C; } unittest { { // Obj test Model model = ConvertObj("./test/sponza.obj"); } { MatProp prop = GetMatProp("Ka"); } }