module dlib.packer; import includes; import dlib.util; import dlib.aliases; import dlib.assets; import dlib.platform; import dlib.alloc; import std.stdio; import std.string; import std.file; import std.path; import std.traits; import std.algorithm.comparison; import core.memory; import std.conv; import std.array; AssetType[string] Lookup = [ ".obj": AT.Model, ".png": AT.Texture, ".jpg": AT.Texture, ".spv": AT.Shader, ]; enum MatProp { None, Ambient, Albedo, Specular, SpecularExp, Dissolve, // Transparency 1.0 -> opaque Transparency, // Transparency 0.0 -> opaque Transmission, OpticalDensity, Illumination, AmbientMap, AlbedoMap, SpecularMap, SpecularHighlightMap, AlphaMap, BumpMap, DisplacementMap, Stencil, Roughness, RoughnessMap, Metallic, MetallicMap, Sheen, SheenMap, ClearcoatThickness, ClearcoatRoughness, Emissive, EmissiveMap, Anisotropy, AnisotropyMap, NormalMap, } struct PackedString { StringHeader header; string str; alias header this; } struct Texture { string name; u8[] data; u32 w; u32 h; u32 ch; } union MeshIdx { struct { u32 v, uv, n, t; }; u32[4] arr; } struct DirEntry { string dir; string[] files; DirEntry[] sub_dirs; } u64 g_asset_count = 0; Texture[] g_model_textures = []; /************************************************** ****** UPDATE FILE_VERSION AFTER CHANGES !! ****** **************************************************/ void main(string[] argv) { string assets_dir = null; string asset_pack_path = null; for(u64 i = 0; i < argv.length; i += 1) { if(argv[i] == "-out" && i+1 < argv.length) { if(!exists(argv[i+1])) { mkdir(argv[i+1]); } if(!isDir(argv[i+1])) { assert(false, "Out directory is not a directory"); } asset_pack_path = argv[i+1]; i += 1; continue; } if(argv[i] == "-assets") { if(!exists(argv[i+1])) { assert(false, "Assets directory not found"); } assets_dir = argv[i+1]; i += 1; continue; } } if(asset_pack_path != null) { if(!Cwd!(string)().endsWith("build")) { if(!exists("build")) { mkdir("build"); } if(exists("build") && isDir("build")) { assets_pack_path = "./build/assets.sgp"; } else assert(false, "Unable to make default build directory"); } } if(assets_dir == null) { assert(false, "No assets directory provided"); } if(pack) { PackFile(assets_pack_path, assets_dir); debug TestFile(assets_pack_path, assets_dir); } } void ScanDir(DirEntry* entry, string dir) { string cwd = Cwd!(string)(); foreach(string e; dirEntries(dir, SpanMode.shallow)) { if(e == ".." || e == ".") continue; if(isDir(e)) { entry.sub_dirs ~= DirEntry(e, []. []); } if(isFile(e)) { entry.files ~= e; } } for(u64 i = 0; i < entry.sub_dirs.length; i += 1) { ScanDir(&entry.sub_dirs[i], entry.sub_dirs[i].dir); } } void PackFile(string file_path, string assets_dir) { string cwd = Cwd!(string)(); File ap = File(file_path, "wb"); ChDir(assets_dir); DirEntry base_dir = [ { dir: "", files: [], sub_dirs: [], }, ]; ScanDir(&base_dir, "."); FileHeader h = InitHeader(g_asset_count); ap.rawWrite([h]); u64 offset = FileHeader.sizeof + (AssetInfo.sizeof * g_asset_count); AssetInfo[] asset_info; foreach(file; g_file_names) { AssetType type = AT.None; foreach(extension, t; Lookup) { if (file.endsWith(extension)) { type = t; break; } } assert(type != AT.None, "Asset Type is none, offending file " ~ file); auto f = File(file, "rb"); u64 length = cast(u64)f.size(); string base_name = chompPrefix(file, "./"); AssetInfo info = { hash: Hash(base_name), offset: offset, length: length, type: type, }; auto data = f.rawRead(new u8[length]); assert(length == data.length, "rawRead failure: data length returned doesn't match"); ap.seek(offset); ap.rawWrite(data); offset += length; asset_info ~= info; f.close(); } ap.seek(FileHeader.sizeof); ap.rawWrite(asset_info); ap.flush(); ap.close(); ChDir(cwd); } void TestFile(string file_path, string assets_dir) { File ap = File(file_path, "rb"); scope(exit) { ap.flush(); ap.close(); } FileHeader file_header = ap.rawRead(new FileHeader[1])[0]; FileHeader test_header = InitHeader(g_asset_count); assert(file_header == test_header, "TestFile failure: Header is incorrect"); AssetInfo[] file_info = ap.rawRead(new AssetInfo[g_asset_count]); assert(file_info.length == file_header.asset_count, "TestFile failure: Incorrect AssetInfo length returned"); u64 asset_index = 0; foreach(i, file; g_file_names) { scope(exit) asset_index += 1; AssetInfo* info = file_info.ptr + asset_index; File asset = File(file, "rb"); u8[] data = asset.rawRead(new u8[asset.size()]); assert(data.length == info.length, "TestFile failure: File length read is incorrect"); string base_name = chompPrefix(file, "./"); assert(Hash(base_name) == info.hash, "TestFile failure: File hash is incorrect"); ap.seek(info.offset); u8[] pack_data = ap.rawRead(new u8[info.length]); assert(equal!((a, b) => a == b)(data[], pack_data[]), "TestFile failure: Asset data does not match file data"); } } MatProp GetMatProp(string str) { switch(str) with(MatProp) { // Vec3 case "Ka": return Ambient; case "Kd": return Albedo; case "Ks": return Specular; case "Tf": return Transmission; case "Ke": return Emissive; // Illum case "illum": return Illumination; // string case "map_Ka": return AmbientMap; case "map_Kd": return AlbedoMap; case "map_Ks": return SpecularMap; case "map_Ns": return SpecularHighlightMap; case "map_d": return AlphaMap; case "map_bump": case "bump": return BumpMap; case "map_Pr": return RoughnessMap; case "map_Pm": return MetallicMap; case "map_Ke": return EmissiveMap; case "map_Ps": return SheenMap; case "norm": return NormalMap; case "anisor": return AnisotropyMap; case "disp": return DisplacementMap; case "decal": return Stencil; // f32 case "Ns": return SpecularExp; case "d": return Dissolve; case "Tr": return Transparency; case "Ni": return OpticalDensity; case "Pr": return Roughness; case "Pm": return Metallic; case "Pc": return ClearcoatThickness; case "Pcr": return ClearcoatRoughness; case "aniso": return Anisotropy; case "Ps": return Sheen; default: return None; } } MatColor GetMatColor(MatProp prop) { switch(prop) with(MatProp) { case Ambient: return MatColor.Ambient; case Albedo: return MatColor.Albedo; case Specular: return MatColor.Specular; case Transmission: return MatColor.Transmission; case Emissive: return MatColor.Emissive; default: assert(false, "Unknown MatProp to MatColor conversion"); } } MatFloat GetMatFloat(MatProp prop) { switch(prop) with(MatProp) { case SpecularExp: return MatFloat.SpecularExp; case Transparency: case Dissolve: return MatFloat.Alpha; case OpticalDensity: return MatFloat.OpticalDensity; case Roughness: return MatFloat.Roughness; case Metallic: return MatFloat.Metallic; case ClearcoatThickness: return MatFloat.ClearcoatThickness; case ClearcoatRoughness: return MatFloat.ClearcoatRoughness; case Anisotropy: return MatFloat.Anisotropy; case Sheen: return MatFloat.Sheen; default: assert(false, "Unknown MatProp to MatFloat conversion"); } } MatMap GetMatMap(MatProp prop) { switch(prop) with(MatProp) { case AmbientMap: return MatMap.Ambient; case AlbedoMap: return MatMap.Albedo; case SpecularMap: return MatMap.Specular; case SpecularHighlightMap: return MatMap.SpecularHighlight; case AlphaMap: return MatMap.Alpha; case BumpMap: return MatMap.Bump; case RoughnessMap: return MatMap.Roughness; case MetallicMap: return MatMap.Metallic; case EmissiveMap: return MatMap.Emissive; case SheenMap: return MatMap.Sheen; case NormalMap: return MatMap.Normal; case AnisotropyMap: return MatMap.Anisotropy; case DisplacementMap: return MatMap.Displacement; case Stencil: return MatMap.Ambient; default: assert(false, "Unknown MatProp to MatMap conversion"); } } MatMap GetMatMap(string str) { return GetMatMap(GetMatProp(str)); } MatFloat GetMatFloat(string str) { return GetMatFloat(GetMatProp(str)); } MatColor GetMatColor(string str) { return GetMatColor(GetMatProp(str)); } static u32 MagicValue(string str) { assert(str.length == 4, "Magic value must 4 characters"); return cast(u32)(cast(u32)(str[0] << 24) | cast(u32)(str[1] << 16) | cast(u32)(str[2] << 8) | cast(u32)(str[3] << 0)); } static FileHeader InitHeader(u64 asset_count) { FileHeader header = { magic: MagicValue("steg"), file_version: FILE_VERSION, asset_count: asset_count, asset_info_offset: FileHeader.sizeof, }; return header; } string[][] TokenizeLines(u8[] data) { string[][] tokens = []; string[] line_tokens = []; u64 start = -1; for(u64 i = 0; i < data.length; i += 1) { if(i64(start) != -1 && CheckWhiteSpace(data[i])) { line_tokens ~= ConvString(data[start .. i]); start = -1; } if(data[i] == '\n') { tokens ~= line_tokens; line_tokens = []; continue; } if(i64(start) == -1 && !CheckWhiteSpace(data[i])) { start = i; continue; } } return tokens; } u8[] OpenFile(string file_name) { File f; u8[] data; try { f = File(file_name, "rb"); data = new u8[f.size()]; f.rawRead(data); } catch(Exception e) { data = null; } f.close(); return data; } /* Model ConvertObj(string file_name) { // TODO: // - Deduplicate vertices u8[] data = OpenFile(file_name); Model model; u64 vcount, uvcount, ncount, fcount, gcount, mcount; string[][] tokens = TokenizeLines(data); for(u64 i = 0; i < tokens.length; i += 1) { if(tokens[i].length == 0) continue; switch(tokens[i][0]) { case "v": vcount += 1; break; case "vt": uvcount += 1; break; case "vn": ncount += 1; break; case "f": fcount += 1; break; case "g": gcount += 1; break; case "usemtl": mcount += 1; break; default: break; } } Vec3[] positions = new Vec3[vcount]; Vec3[] normals = new Vec3[ncount]; Vec2[] uvs = new Vec2[uvcount]; MeshIdx[3][][] idx = []; MaterialData[] mtls = []; vcount = 0; ncount = 0; uvcount = 0; MeshIdx[3][] part_idx = []; for(u64 i = 0; i < tokens.length; i += 1) { if(tokens[i][0] == "#") continue; if(tokens[i][0] == "v") { if(tokens[i].length < 4) assert(false, "OBJ file error, not enough points for case [v]"); positions[vcount++] = Vec3(ToF32(tokens[i][1]), ToF32(tokens[i][2]), ToF32(tokens[i][3])); continue; } if(tokens[i][0] == "vn") { if(tokens[i].length < 4) assert(false, "OBJ file error, not enough points for case [vn]"); normals[ncount++] = Vec3(ToF32(tokens[i][1]), ToF32(tokens[i][2]), ToF32(tokens[i][3])); continue; } if(tokens[i][0] == "vt") { if(tokens[i].length < 3) assert(false, "OBJ file error, not enough points for case [vt]"); uvs[uvcount++] = Vec2(ToF32(tokens[i][1]), ToF32(tokens[i][2])); continue; } if(tokens[i][0] == "f") { u32 sep_count = StrCharCount(tokens[i][1], '/'); if(tokens[i].length == 4) { MeshIdx[3] face; for(u64 j = 1; j < tokens[i].length; j += 1) { string[] parts = tokens[i][j].split('/'); if(sep_count == 0) { face[j-1] = MeshIdx(v: to!u32(parts[0])); } if(sep_count == 1) { face[j-1] = MeshIdx(v: to!u32(parts[0]), uv: to!u32(parts[1])); } if(sep_count == 2) { MeshIdx mesh_idx; foreach(ipart, part; parts) { if(part == "") continue; mesh_idx.arr[ipart] = to!u32(part); } face[j-1] = mesh_idx; } } part_idx ~= face; } else assert(false, "Only triangles or quads supported for mesh face"); continue; } if(tokens[i][0] == "g" && part_idx.length > 0) { idx ~= part_idx; part_idx = []; continue; } if(tokens[i][0] == "mtllib") { u8[] mtl_data = OpenFile(GetFilePath(file_name) ~ tokens[i][1]); MaterialData* mtl = null; string[][] mtl_tokens = TokenizeLines(mtl_data); for(u64 j = 0; j < mtl_tokens.length; j += 1) { if(mtl_tokens[j].length == 0) { if(mtl) { mtls ~= *mtl; mtl = null; } continue; } if(mtl_tokens[j][0] == "newmtl") { mtl = new MaterialData; mtl.name = mtl_tokens[j][1]; continue; } if(!mtl) continue; switch(mtl_tokens[j][0]) { case "Ka", "Kd", "Ks", "Tf", "Ke": { mtl.colors[GetMatColor(mtl_tokens[j][0])] = Vec3(ToF32(mtl_tokens[j][1]), ToF32(mtl_tokens[j][2]), ToF32(mtl_tokens[j][3])); } break; case "Ns", "d", "Tr", "Ni", "Pr", "Pm", "Pc", "Pcr", "aniso", "Ps": { f32 v = ToF32(mtl_tokens[j][1]); if(mtl_tokens[j][1] == "Tr") { v = 1.0 - v; } mtl.props[GetMatFloat(mtl_tokens[j][0])] = v; } break; case "map_Ka", "map_Kd", "map_Ks", "map_Ns", "map_d", "map_bump", "anisor", "bump", "map_Pr", "map_Pm", "map_Ke", "map_Ps", "norm", "disp", "decal": { mtl.maps[GetMatMap(mtl_tokens[j][0])] = mtl_tokens[j][1]; } break; case "illum": { mtl.illum = cast(IllumModel)(to!(u32)(mtl_tokens[j][1])); } break; default: break; } } if(mtl) { mtls ~= *mtl; mtl = null; } } } u64 face_count; foreach(part; idx) { face_count += part.length; } ModelData md = { name: baseName(file_name, ".obj"), meshes: [ { vtx: new Vertex[face_count*3] }, ] }; Logf("%s %s", (Vertex.sizeof * md.vtx.length), (positions.length*Vec3.sizeof + uvs.length*Vec2.sizeof + normals.length*Vec3.sizeof)); u64 vtx_count = 0; foreach(part; idx) { for(u64 i = 0; i < part.length; i += 1) { for(u64 j = 0; j < 3; j += 1) { MeshIdx* mi = part[i].ptr + j; if(mi.v ) md.meshes[0].vtx[vtx_count+j].pos = positions[mi.v-1]; if(mi.n ) md.meshes[0].vtx[vtx_count+j].normal = normals[mi.n-1]; if(mi.uv) md.meshes[0].vtx[vtx_count+j].uv = uvs[mi.uv-1]; } vtx_count += 3; } } return model; } */ pragma(inline) f32 ToF32(string str) { return to!(f32)(str); } pragma(inline) string ConvString(u8[] bytes) { return (cast(immutable(char)*)bytes.ptr)[0 .. bytes.length]; } pragma(inline) bool CheckWhiteSpace(u8 ch) { return ch == ' ' || ch == '\t'|| ch == '\n'|| ch == 0x0D|| ch == 0x0A|| ch == 0x0B|| ch == 0x0C; } unittest { { // Obj test //Model model = ConvertObj("./test/sponza.obj"); } { MatProp prop = GetMatProp("Ka"); } }