diff --git a/.gitmodules b/.gitmodules index 4b94285..bad792b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ -[submodule "src/DLibs"] - path = src/DLibs - url = https://git.sleepy.day/sleepy-day/DLibs.git [submodule "src/VulkanRenderer"] path = src/VulkanRenderer url = https://git.sleepy.day/sleepy-day/VulkanRenderer.git +[submodule "src/dlib"] + path = src/dlib + url = https://git.sleepy.day/sleepy-day/dlib.git diff --git a/build.sh b/build.sh index 6bde3b5..3bfa50a 100755 --- a/build.sh +++ b/build.sh @@ -28,4 +28,4 @@ done /bin/bash src/VulkanRenderer/build.sh build -/bin/bash src/DLibs/build.sh build +/bin/bash src/dlib/build.sh build diff --git a/dub.json b/dub.json index 60c0d30..118cea4 100644 --- a/dub.json +++ b/dub.json @@ -9,8 +9,8 @@ "targetPath": "build", "sourceFiles-linux": ["build/libvma.a", "build/libstb.a", "build/libm3d.a", "build/libcglm.a"], "sourceFiles-windows": [], - "importPaths": ["src/editor", "src/DLibs", "src/DLibs/external/xxhash", "src/VulkanRenderer"], - "sourcePaths": ["src/editor", "src/DLibs", "src/DLibs/external/xxhash", "src/VulkanRenderer"], + "importPaths": ["src/editor", "src/dlib", "src/dlib/external/xxhash", "src/VulkanRenderer"], + "sourcePaths": ["src/editor", "src/dlib", "src/dlib/external/xxhash", "src/VulkanRenderer"], "libs-linux": ["xcb", "X11", "X11-xcb", "vulkan", "stdc++", "xcb-xfixes", "freetype"], "libs-windows": [], "preGenerateCommands-linux": ["./build.sh"], diff --git a/src/DLibs b/src/DLibs deleted file mode 160000 index 79d128c..0000000 --- a/src/DLibs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 79d128c872e10075e29d19616645d82cb53e3450 diff --git a/src/dlib b/src/dlib new file mode 160000 index 0000000..98701be --- /dev/null +++ b/src/dlib @@ -0,0 +1 @@ +Subproject commit 98701be20c223d1130a1f423c8ca3b520c9d4a95 diff --git a/src/editor/buffer.d b/src/editor/buffer.d index b412b32..424f8c7 100644 --- a/src/editor/buffer.d +++ b/src/editor/buffer.d @@ -1,8 +1,9 @@ -import aliases; -import math; -import alloc; -import util; +import dlib.aliases; +import dlib.math; +import dlib.alloc; +import dlib.util; import core.stdc.stdio : EOF; +import parsing; struct FlatBuffer { @@ -329,660 +330,6 @@ PeekNextChar(FlatBuffer* fb) return fb.tk.pos+1 < fb.length ? fb.data[fb.tk.pos+1] : 0; } -const const(u8[])[]['z'-95] D_KEYWORDS = [ - '_'-95: [ - cast(u8[])r"__FILE__", - cast(u8[])r"__FILE_FULL_PATH__", - cast(u8[])r"__FUNCTION__", - cast(u8[])r"__LINE__", - cast(u8[])r"__MODULE__", - cast(u8[])r"__PRETTY_FUNCTION__", - cast(u8[])r"__gshared", - cast(u8[])r"__parameters", - cast(u8[])r"__rvalue", - cast(u8[])r"__traits", - cast(u8[])r"__vector", - ], - 'a'-95: [ - cast(u8[])r"abstract", - cast(u8[])r"alias", - cast(u8[])r"align", - cast(u8[])r"asm", - cast(u8[])r"assert", - cast(u8[])r"auto", - ], - 'b'-95: [ - cast(u8[])r"bool", - cast(u8[])r"break", - cast(u8[])r"byte", - ], - 'c'-95: [ - cast(u8[])r"case", - cast(u8[])r"cast", - cast(u8[])r"catch", - cast(u8[])r"char", - cast(u8[])r"class", - cast(u8[])r"const", - cast(u8[])r"continue", - ], - 'd'-95: [ - cast(u8[])r"dchar", - cast(u8[])r"debug", - cast(u8[])r"default", - cast(u8[])r"delegate", - cast(u8[])r"deprecated", - cast(u8[])r"do", - cast(u8[])r"double", - ], - 'e'-95: [ - cast(u8[])r"else", - cast(u8[])r"enum", - cast(u8[])r"export", - cast(u8[])r"extern", - ], - 'f'-95: [ - cast(u8[])r"false", - cast(u8[])r"final", - cast(u8[])r"finally", - cast(u8[])r"float", - cast(u8[])r"for", - cast(u8[])r"foreach", - cast(u8[])r"foreach_reverse", - cast(u8[])r"function", - ], - 'g'-95: [ - cast(u8[])r"goto", - ], - 'i'-95: [ - cast(u8[])r"if", - cast(u8[])r"immutable", - cast(u8[])r"import", - cast(u8[])r"in", - cast(u8[])r"inout", - cast(u8[])r"int", - cast(u8[])r"interface", - cast(u8[])r"invariant", - cast(u8[])r"is", - ], - 'l'-95: [ - cast(u8[])r"lazy", - cast(u8[])r"long", - ], - 'm'-95: [ - cast(u8[])r"mixin", - cast(u8[])r"module", - ], - 'n'-95: [ - cast(u8[])r"new", - cast(u8[])r"nothrow", - cast(u8[])r"null", - ], - 'o'-95: [ - cast(u8[])r"out", - cast(u8[])r"override", - ], - 'p'-95: [ - cast(u8[])r"package", - cast(u8[])r"pragma", - cast(u8[])r"private", - cast(u8[])r"protected", - cast(u8[])r"public", - cast(u8[])r"pure", - ], - 'r'-95: [ - cast(u8[])r"real", - cast(u8[])r"ref", - cast(u8[])r"return", - ], - 's'-95: [ - cast(u8[])r"scope", - cast(u8[])r"shared", - cast(u8[])r"short", - cast(u8[])r"static", - cast(u8[])r"struct", - cast(u8[])r"super", - cast(u8[])r"switch", - cast(u8[])r"synchronized", - ], - 't'-95: [ - cast(u8[])r"template", - cast(u8[])r"this", - cast(u8[])r"throw", - cast(u8[])r"true", - cast(u8[])r"try", - cast(u8[])r"typeid", - cast(u8[])r"typeof", - ], - 'u'-95: [ - cast(u8[])r"ubyte", - cast(u8[])r"uint", - cast(u8[])r"ulong", - cast(u8[])r"union", - cast(u8[])r"unittest", - cast(u8[])r"ushort", - ], - 'v'-95: [ - cast(u8[])r"version", - cast(u8[])r"void", - ], - 'w'-95: [ - cast(u8[])r"wchar", - cast(u8[])r"while", - cast(u8[])r"with", - ], -]; - -void -Tokenize(FlatBuffer* fb) -{ - Tokenizer* tk = &fb.tk; - - u64 semi_count; - foreach(i; 0 .. fb.length) - { - if (fb.data.ptr[i] == ';') - { - semi_count += 1; - } - } - - Reset(&tk.arena); - - tk.tokens = AllocArray!(Token)(&tk.arena, fb.length); - tk.tk_count = 1; - - bool _macro = false; - bool id_of_line = false; - Token* prev = tk.tokens.ptr; - while (true) - { - scope(exit) tk.tk_count += 1; - Token* t = NextToken(fb); - if (t == null) break; - - switch(t.type) - { - case TT.Identifier: - case TT.Keyword: - { - TokenStyle ts = t.type == TT.Keyword ? TS.Keyword : TS.Identifier; - tk.buffer[t.start .. t.end] = ts; - } break; - case TT.Number: - { - tk.buffer[t.start .. t.end] = TS.Number; - } break; - case TT.Comment: - { - tk.buffer[t.start .. t.end] = TS.Comment; - } break; - case TT.DoubleQuote: - case TT.BackTick: - { - tk.buffer[t.start .. t.end] = TS.String; - } break; - case TT.SingleQuote: - { - tk.buffer[t.start .. t.end] = TS.Char; - } break; - case TT.LeftBracket: - { - if (prev != null && _macro && prev.type != TT.Exclamation) - { - _macro = false; - } - } goto case TT.Comma; - case TT.LeftBrace: - { - if (prev) - { - if (prev.type == TT.RightBracket) - { - i64 i = tk.tk_count - 2; - Token* pt = null; - bool is_macro = false; - i64 brackets = 1; - while (i > 0) - { - pt = tk.tokens.ptr + i; - i -= 1; - - if (pt.type == TT.Keyword && brackets == 0) break; - - if (pt.type == TT.LeftBracket) - { - brackets -= 1; - continue; - } - - if (pt.type == TT.RightBracket) - { - brackets += 1; - is_macro = true; - continue; - } - - if (pt.type == TT.Identifier && brackets == 0) - { - if (i >= 0) - { - Token* tmp = tk.tokens.ptr + i; - tmp.type = TT.Type; - tk.buffer[tmp.start .. tmp.end] = TS.Type; - } - - pt.type = TT.Function; - tk.buffer[pt.start .. pt.end] = TS.Function; - - brackets = 0; - Token* ppt = null; - while (i < tk.tk_count - 1) - { - ppt = pt; - i += 1; - pt = tk.tokens.ptr + i; - - bool id_keyword = pt.type == TT.Identifier || pt.type == TT.Keyword; - - if ( is_macro && brackets == 4) break; - if (!is_macro && brackets == 2) break; - - if (is_macro && brackets < 2 && id_keyword) - { - pt.type = TT.Type; - tk.buffer[pt.start .. pt.end] = TS.Type; - continue; - } - - if (((!is_macro && brackets < 2) || (is_macro && brackets == 3)) && id_keyword && (ppt.type == TT.LeftBracket || ppt.type == TT.Comma)) - { - pt.type = TT.Type; - tk.buffer[pt.start .. pt.end] = TS.Type; - continue; - } - - if (pt.type == TT.LeftBracket || pt.type == TT.RightBracket) - { - brackets += 1; - continue; - } - } - - break; - } - } - } - } - } goto case TT.Comma; - case TT.Semicolon: - { - id_of_line = false; - } goto case TT.Comma; - case TT.RightBracket: - case TT.RightBrace: - case TT.LeftSquareBrace: - case TT.RightSquareBrace: - case TT.Colon: - case TT.Equals: - case TT.Comma: - { - tk.buffer[t.start .. t.end] = TS.Bracket; - } break; - case TT.Exclamation: - { - _macro = true; - } goto case TT.Equals; - case TT.Tilde: - case TT.Question: - case TT.LessThan: - case TT.GreaterThan: - case TT.Dollar: - case TT.Percent: - case TT.Caret: - case TT.Ampersand: - case TT.Asterisk: - case TT.VertBar: - case TT.Plus: - case TT.Minus: - case TT.Slash: - { - tk.buffer[t.start .. t.end] = TS.Op; - } break; - default: - tk.buffer[t.start .. t.end] = TS.Keyword; - } - - prev = t; - } -} - -Token* -PrevToken(FlatBuffer* fb, i64 prev) -{ - i64 c = cast(i64)(fb.tk.tk_count); - return (c-prev) > 0 ? fb.tk.tokens.ptr + (c-prev) : fb.tk.tokens.ptr + 0; -} - -Token* -NextToken(FlatBuffer* fb) -{ - Tokenizer* tk = &fb.tk; - - SkipWhiteSpace(fb); - - u8 ch = fb.data[tk.pos]; - - Token* t = null; - if (tk.pos < fb.length) - { - t = tk.tokens.ptr + tk.tk_count; - t.start = tk.pos; - - bool ascii = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'); - - if (ch == 'r') - { - u8 next = PeekNextChar(fb); - if (next == '"') - { - tk.pos += 1; - ParseStr(fb); - } - else - { - ParseId(fb); - } - } - else if (ch >= '0' && ch <= '9') - { - ParseNum(fb); - } - else if (ch == 'P' || ch == 'E' || ch == 'e' || ch == 'p') - { - u8 next = PeekNextChar(fb); - if (next == '-' || next == '+' || (next >= '0' && next <= '9')) - { - ParseNum(fb); - } - else - { - ParseId(fb); - } - } - else if (ascii || ch == '_') - { - ParseId(fb); - } - else - { - switch (ch) - { - case '$': t.type = TT.Dollar; goto case u8.max; - case '(': t.type = TT.LeftBracket; goto case u8.max; - case ')': t.type = TT.RightBracket; goto case u8.max; - case '[': t.type = TT.LeftSquareBrace; goto case u8.max; - case ']': t.type = TT.RightSquareBrace; goto case u8.max; - case '{': t.type = TT.LeftBrace; goto case u8.max; - case '}': t.type = TT.RightBrace; goto case u8.max; - case '|': t.type = TT.VertBar; goto case u8.max; - case ';': t.type = TT.Semicolon; goto case u8.max; - case ':': t.type = TT.Colon; goto case u8.max; - case '?': t.type = TT.Question; goto case u8.max; - case ',': t.type = TT.Comma; goto case u8.max; - case '#': t.type = TT.Hash; goto case u8.max; - case '~': t.type = TT.Tilde; goto case u8.max; - case '%': t.type = TT.Percent; goto case 251; - case '^': t.type = TT.Caret; goto case 251; - case '&': t.type = TT.Ampersand; goto case 251; - case '*': t.type = TT.Asterisk; goto case 251; - case '-': t.type = TT.Minus; goto case 251; - case '=': t.type = TT.Equals; goto case 251; - case '+': t.type = TT.Plus; goto case 251; - case '<': t.type = TT.LessThan; goto case 251; - case '>': t.type = TT.GreaterThan; goto case 251; - - case '`': - { - t.type = TT.BackTick; - tk.pos += 1; - while (fb.data[tk.pos] != '`' && tk.pos < fb.length) - { - tk.pos += 1; - } - t.end = tk.pos+1; - } break; - case '\'': - { - t.type = TT.SingleQuote; - - bool ignore = true; - while ((fb.data[tk.pos] != '\'' || ignore) && tk.pos < fb.length) - { - tk.pos += 1; - ignore = fb.data[tk.pos] == '\\'; - } - - tk.pos += 1; - t.end = tk.pos; - } break; - case '"': - ParseStr(fb); - break; - case '!': - t.type = TT.Exclamation; - goto case 251; - case '@': - { - t.type = TT.At; - while (tk.pos < fb.length) - { - tk.pos += 1; - u8 c = fb.data[tk.pos]; - - if (CheckWhiteSpace(c)) break; - if (!(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z')) break; - } - - t.end = tk.pos; - } break; - case '.': - { - t.type = TT.Dot; - while (tk.pos < fb.length) - { - tk.pos += 1; - if (tk.pos != '.') break; - } - t.end = tk.pos; - } break; - case '/': - { - t.type = TT.Slash; - u8 next = PeekNextChar(fb); - if (next == '/') - { - tk.pos += 1; - t.type = TT.Comment; - - while (tk.pos < fb.length) - { - tk.pos += 1; - - if (CheckEOL(fb.data[tk.pos])) - { - break; - } - } - - t.end = tk.pos+1; - } - else - { - goto case u8.max; - } - } break; - case 251: - { - u8 next = PeekNextChar(fb); - if (next == '=') - { - tk.pos += 1; - } - - t.end = tk.pos+1; - tk.pos += 1; - } break; - case u8.max: - { - t.end = tk.pos+1; - tk.pos += 1; - } break; - default: - t.type = TT.None; - t.end = tk.pos+1; - tk.pos += 1; - } - } - } - - return t; -} - -void -ParseNum(FlatBuffer* fb) -{ - Tokenizer* tk = &fb.tk; - - Token* t = tk.tokens.ptr + tk.tk_count; - t.type = TT.Number; - t.start = tk.pos; - - bool first = true; - while (tk.pos < fb.length) - { - tk.pos += 1; - - u8 ch = fb.data[tk.pos]; - - if (ch != '_' && ch != '.' && !(ch >= '0' && ch <= '9') && ch != 'x' && ch != 'X' && !((ch == '-' || ch == '+') && first)) - { - break; - } - - first = false; - } - - while (tk.pos < fb.length) - { - u8 ch = fb.data[tk.pos]; - if (ch != 'l' && ch != 'u' && ch != 'U' && ch != 'L' && ch != 'f' && ch != 'F') - { - break; - } - - tk.pos += 1; - } - - t.end = tk.pos; -} - -void -ParseStr(FlatBuffer* fb) -{ - Tokenizer* tk = &fb.tk; - - tk.tokens[tk.tk_count].type = TT.DoubleQuote; - - bool ignore = true; - while ((fb.data[tk.pos] != '"' || ignore) && tk.pos < fb.length) - { - tk.pos += 1; - ignore = fb.data[tk.pos] == '\\'; - } - - tk.pos += 1; - tk.tokens[tk.tk_count].end = tk.pos; -} - -void -ParseId(FlatBuffer* fb) -{ - Tokenizer* tk = &fb.tk; - Token* t = tk.tokens.ptr + tk.tk_count; - - t.type = TT.Identifier; - t.start = tk.pos; - - while (tk.pos < fb.length) - { - tk.pos += 1; - - u8 ch = fb.data[tk.pos]; - - if (!(ch >= 'a' && ch <= 'z') && !(ch >= 'A' && ch <= 'Z') && ch != '.' && ch != '_' && !(ch >= '0' && ch <= '9')) - { - break; - } - } - - t.end = tk.pos; - - u8 ch = cast(u8)(fb.data[t.start] - cast(u8)(95)); // Index into keywords lookup - if (ch < D_KEYWORDS.length && D_KEYWORDS[ch].length > 0) - { - u8[] id = fb.data[t.start .. t.end]; - foreach(k; D_KEYWORDS[ch]) - { - if (id == k) - { - t.type = TT.Keyword; - break; - } - } - } - - assert(t.start < t.end, "ParseId failure: start is lower or equal to end"); -} - -pragma(inline): bool -CheckEOL(u8 ch) -{ - return ch == 0x0D|| - ch == 0x0A|| - ch == 0x0B|| - ch == 0x0C; -} - -pragma(inline): bool -CheckWhiteSpace(u8 ch) -{ - return ch == ' ' || - ch == '\t'|| - ch == '\n'|| - ch == 0x0D|| - ch == 0x0A|| - ch == 0x0B|| - ch == 0x0C; -} - -void -SkipWhiteSpace(FlatBuffer* fb) -{ - while (fb.tk.pos < fb.length) - { - u8 ch = fb.data[fb.tk.pos]; - - bool white_space = CheckWhiteSpace(ch); - - if (!white_space) - { - break; - } - - fb.tk.pos += 1; - } -} - /* bool Advance(FlatBuffer* fb) diff --git a/src/editor/editor.d b/src/editor/editor.d index 94ecf6e..43a8100 100644 --- a/src/editor/editor.d +++ b/src/editor/editor.d @@ -1,10 +1,10 @@ -import aliases; -import util; -import platform; -import fonts; +import dlib.aliases; +import dlib.util; +import dlib.platform; +import dlib.fonts; import vulkan; -import math; -import alloc; +import dlib.math; +import dlib.alloc; import buffer; import std.stdio; diff --git a/src/editor/main.d b/src/editor/main.d index fd808be..d0480a7 100644 --- a/src/editor/main.d +++ b/src/editor/main.d @@ -1,8 +1,8 @@ -import aliases; -import platform; +import dlib.aliases; +import dlib.platform; import editor; import std.stdio; -import alloc; +import dlib.alloc; void main(string[] argv) { diff --git a/src/editor/parsing.d b/src/editor/parsing.d new file mode 100644 index 0000000..5384b87 --- /dev/null +++ b/src/editor/parsing.d @@ -0,0 +1,659 @@ +import dlib.aliases; +import dlib.alloc; + +import buffer; + +const const(u8[])[]['z'-95] D_KEYWORDS = [ + '_'-95: [ + cast(u8[])r"__FILE__", + cast(u8[])r"__FILE_FULL_PATH__", + cast(u8[])r"__FUNCTION__", + cast(u8[])r"__LINE__", + cast(u8[])r"__MODULE__", + cast(u8[])r"__PRETTY_FUNCTION__", + cast(u8[])r"__gshared", + cast(u8[])r"__parameters", + cast(u8[])r"__rvalue", + cast(u8[])r"__traits", + cast(u8[])r"__vector", + ], + 'a'-95: [ + cast(u8[])r"abstract", + cast(u8[])r"alias", + cast(u8[])r"align", + cast(u8[])r"asm", + cast(u8[])r"assert", + cast(u8[])r"auto", + ], + 'b'-95: [ + cast(u8[])r"bool", + cast(u8[])r"break", + cast(u8[])r"byte", + ], + 'c'-95: [ + cast(u8[])r"case", + cast(u8[])r"cast", + cast(u8[])r"catch", + cast(u8[])r"char", + cast(u8[])r"class", + cast(u8[])r"const", + cast(u8[])r"continue", + ], + 'd'-95: [ + cast(u8[])r"dchar", + cast(u8[])r"debug", + cast(u8[])r"default", + cast(u8[])r"delegate", + cast(u8[])r"deprecated", + cast(u8[])r"do", + cast(u8[])r"double", + ], + 'e'-95: [ + cast(u8[])r"else", + cast(u8[])r"enum", + cast(u8[])r"export", + cast(u8[])r"extern", + ], + 'f'-95: [ + cast(u8[])r"false", + cast(u8[])r"final", + cast(u8[])r"finally", + cast(u8[])r"float", + cast(u8[])r"for", + cast(u8[])r"foreach", + cast(u8[])r"foreach_reverse", + cast(u8[])r"function", + ], + 'g'-95: [ + cast(u8[])r"goto", + ], + 'i'-95: [ + cast(u8[])r"if", + cast(u8[])r"immutable", + cast(u8[])r"import", + cast(u8[])r"in", + cast(u8[])r"inout", + cast(u8[])r"int", + cast(u8[])r"interface", + cast(u8[])r"invariant", + cast(u8[])r"is", + ], + 'l'-95: [ + cast(u8[])r"lazy", + cast(u8[])r"long", + ], + 'm'-95: [ + cast(u8[])r"mixin", + cast(u8[])r"module", + ], + 'n'-95: [ + cast(u8[])r"new", + cast(u8[])r"nothrow", + cast(u8[])r"null", + ], + 'o'-95: [ + cast(u8[])r"out", + cast(u8[])r"override", + ], + 'p'-95: [ + cast(u8[])r"package", + cast(u8[])r"pragma", + cast(u8[])r"private", + cast(u8[])r"protected", + cast(u8[])r"public", + cast(u8[])r"pure", + ], + 'r'-95: [ + cast(u8[])r"real", + cast(u8[])r"ref", + cast(u8[])r"return", + ], + 's'-95: [ + cast(u8[])r"scope", + cast(u8[])r"shared", + cast(u8[])r"short", + cast(u8[])r"static", + cast(u8[])r"struct", + cast(u8[])r"super", + cast(u8[])r"switch", + cast(u8[])r"synchronized", + ], + 't'-95: [ + cast(u8[])r"template", + cast(u8[])r"this", + cast(u8[])r"throw", + cast(u8[])r"true", + cast(u8[])r"try", + cast(u8[])r"typeid", + cast(u8[])r"typeof", + ], + 'u'-95: [ + cast(u8[])r"ubyte", + cast(u8[])r"uint", + cast(u8[])r"ulong", + cast(u8[])r"union", + cast(u8[])r"unittest", + cast(u8[])r"ushort", + ], + 'v'-95: [ + cast(u8[])r"version", + cast(u8[])r"void", + ], + 'w'-95: [ + cast(u8[])r"wchar", + cast(u8[])r"while", + cast(u8[])r"with", + ], +]; + +void +Tokenize(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + + u64 semi_count; + foreach(i; 0 .. fb.length) + { + if (fb.data.ptr[i] == ';') + { + semi_count += 1; + } + } + + Reset(&tk.arena); + + tk.tokens = AllocArray!(Token)(&tk.arena, fb.length); + tk.tk_count = 1; + + bool _macro = false; + bool id_of_line = false; + Token* prev = tk.tokens.ptr; + while (true) + { + scope(exit) tk.tk_count += 1; + Token* t = NextToken(fb); + if (t == null) break; + + switch(t.type) + { + case TT.Identifier: + case TT.Keyword: + { + TokenStyle ts = t.type == TT.Keyword ? TS.Keyword : TS.Identifier; + tk.buffer[t.start .. t.end] = ts; + } break; + case TT.Number: + { + tk.buffer[t.start .. t.end] = TS.Number; + } break; + case TT.Comment: + { + tk.buffer[t.start .. t.end] = TS.Comment; + } break; + case TT.DoubleQuote: + case TT.BackTick: + { + tk.buffer[t.start .. t.end] = TS.String; + } break; + case TT.SingleQuote: + { + tk.buffer[t.start .. t.end] = TS.Char; + } break; + case TT.LeftBracket: + { + if (prev != null && _macro && prev.type != TT.Exclamation) + { + _macro = false; + } + } goto case TT.Comma; + case TT.LeftBrace: + { + if (prev) + { + if (prev.type == TT.RightBracket) + { + i64 i = tk.tk_count - 2; + Token* pt = null; + bool is_macro = false; + i64 brackets = 1; + while (i > 0) + { + pt = tk.tokens.ptr + i; + i -= 1; + + if (pt.type == TT.Keyword && brackets == 0) break; + + if (pt.type == TT.LeftBracket) + { + brackets -= 1; + continue; + } + + if (pt.type == TT.RightBracket) + { + brackets += 1; + is_macro = true; + continue; + } + + if (pt.type == TT.Identifier && brackets == 0) + { + if (i >= 0) + { + Token* tmp = tk.tokens.ptr + i; + tmp.type = TT.Type; + tk.buffer[tmp.start .. tmp.end] = TS.Type; + } + + pt.type = TT.Function; + tk.buffer[pt.start .. pt.end] = TS.Function; + + brackets = 0; + Token* ppt = null; + while (i < tk.tk_count - 1) + { + ppt = pt; + i += 1; + pt = tk.tokens.ptr + i; + + bool id_keyword = pt.type == TT.Identifier || pt.type == TT.Keyword; + + if ( is_macro && brackets == 4) break; + if (!is_macro && brackets == 2) break; + + if (is_macro && brackets < 2 && id_keyword) + { + pt.type = TT.Type; + tk.buffer[pt.start .. pt.end] = TS.Type; + continue; + } + + if (((!is_macro && brackets < 2) || (is_macro && brackets == 3)) && id_keyword && (ppt.type == TT.LeftBracket || ppt.type == TT.Comma)) + { + pt.type = TT.Type; + tk.buffer[pt.start .. pt.end] = TS.Type; + continue; + } + + if (pt.type == TT.LeftBracket || pt.type == TT.RightBracket) + { + brackets += 1; + continue; + } + } + + break; + } + } + } + } + } goto case TT.Comma; + case TT.Semicolon: + { + id_of_line = false; + } goto case TT.Comma; + case TT.RightBracket: + case TT.RightBrace: + case TT.LeftSquareBrace: + case TT.RightSquareBrace: + case TT.Colon: + case TT.Equals: + case TT.Comma: + { + tk.buffer[t.start .. t.end] = TS.Bracket; + } break; + case TT.Exclamation: + { + _macro = true; + } goto case TT.Equals; + case TT.Tilde: + case TT.Question: + case TT.LessThan: + case TT.GreaterThan: + case TT.Dollar: + case TT.Percent: + case TT.Caret: + case TT.Ampersand: + case TT.Asterisk: + case TT.VertBar: + case TT.Plus: + case TT.Minus: + case TT.Slash: + { + tk.buffer[t.start .. t.end] = TS.Op; + } break; + default: + tk.buffer[t.start .. t.end] = TS.Keyword; + } + + prev = t; + } +} + +Token* +PrevToken(FlatBuffer* fb, i64 prev) +{ + i64 c = cast(i64)(fb.tk.tk_count); + return (c-prev) > 0 ? fb.tk.tokens.ptr + (c-prev) : fb.tk.tokens.ptr + 0; +} + +Token* +NextToken(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + + SkipWhiteSpace(fb); + + u8 ch = fb.data[tk.pos]; + + Token* t = null; + if (tk.pos < fb.length) + { + t = tk.tokens.ptr + tk.tk_count; + t.start = tk.pos; + + bool ascii = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'); + + if (ch == 'r') + { + u8 next = PeekNextChar(fb); + if (next == '"') + { + tk.pos += 1; + ParseStr(fb); + } + else + { + ParseId(fb); + } + } + else if (ch >= '0' && ch <= '9') + { + ParseNum(fb); + } + else if (ch == 'P' || ch == 'E' || ch == 'e' || ch == 'p') + { + u8 next = PeekNextChar(fb); + if (next == '-' || next == '+' || (next >= '0' && next <= '9')) + { + ParseNum(fb); + } + else + { + ParseId(fb); + } + } + else if (ascii || ch == '_') + { + ParseId(fb); + } + else + { + switch (ch) + { + case '$': t.type = TT.Dollar; goto case u8.max; + case '(': t.type = TT.LeftBracket; goto case u8.max; + case ')': t.type = TT.RightBracket; goto case u8.max; + case '[': t.type = TT.LeftSquareBrace; goto case u8.max; + case ']': t.type = TT.RightSquareBrace; goto case u8.max; + case '{': t.type = TT.LeftBrace; goto case u8.max; + case '}': t.type = TT.RightBrace; goto case u8.max; + case '|': t.type = TT.VertBar; goto case u8.max; + case ';': t.type = TT.Semicolon; goto case u8.max; + case ':': t.type = TT.Colon; goto case u8.max; + case '?': t.type = TT.Question; goto case u8.max; + case ',': t.type = TT.Comma; goto case u8.max; + case '#': t.type = TT.Hash; goto case u8.max; + case '~': t.type = TT.Tilde; goto case u8.max; + case '%': t.type = TT.Percent; goto case 251; + case '^': t.type = TT.Caret; goto case 251; + case '&': t.type = TT.Ampersand; goto case 251; + case '*': t.type = TT.Asterisk; goto case 251; + case '-': t.type = TT.Minus; goto case 251; + case '=': t.type = TT.Equals; goto case 251; + case '+': t.type = TT.Plus; goto case 251; + case '<': t.type = TT.LessThan; goto case 251; + case '>': t.type = TT.GreaterThan; goto case 251; + + case '`': + { + t.type = TT.BackTick; + tk.pos += 1; + while (fb.data[tk.pos] != '`' && tk.pos < fb.length) + { + tk.pos += 1; + } + t.end = tk.pos+1; + } break; + case '\'': + { + t.type = TT.SingleQuote; + + bool ignore = true; + while ((fb.data[tk.pos] != '\'' || ignore) && tk.pos < fb.length) + { + tk.pos += 1; + ignore = fb.data[tk.pos] == '\\'; + } + + tk.pos += 1; + t.end = tk.pos; + } break; + case '"': + ParseStr(fb); + break; + case '!': + t.type = TT.Exclamation; + goto case 251; + case '@': + { + t.type = TT.At; + while (tk.pos < fb.length) + { + tk.pos += 1; + u8 c = fb.data[tk.pos]; + + if (CheckWhiteSpace(c)) break; + if (!(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z')) break; + } + + t.end = tk.pos; + } break; + case '.': + { + t.type = TT.Dot; + while (tk.pos < fb.length) + { + tk.pos += 1; + if (tk.pos != '.') break; + } + t.end = tk.pos; + } break; + case '/': + { + t.type = TT.Slash; + u8 next = PeekNextChar(fb); + if (next == '/') + { + tk.pos += 1; + t.type = TT.Comment; + + while (tk.pos < fb.length) + { + tk.pos += 1; + + if (CheckEOL(fb.data[tk.pos])) + { + break; + } + } + + t.end = tk.pos+1; + } + else + { + goto case u8.max; + } + } break; + case 251: + { + u8 next = PeekNextChar(fb); + if (next == '=') + { + tk.pos += 1; + } + + t.end = tk.pos+1; + tk.pos += 1; + } break; + case u8.max: + { + t.end = tk.pos+1; + tk.pos += 1; + } break; + default: + t.type = TT.None; + t.end = tk.pos+1; + tk.pos += 1; + } + } + } + + return t; +} + +void +ParseNum(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + + Token* t = tk.tokens.ptr + tk.tk_count; + t.type = TT.Number; + t.start = tk.pos; + + bool first = true; + while (tk.pos < fb.length) + { + tk.pos += 1; + + u8 ch = fb.data[tk.pos]; + + if (ch != '_' && ch != '.' && !(ch >= '0' && ch <= '9') && ch != 'x' && ch != 'X' && !((ch == '-' || ch == '+') && first)) + { + break; + } + + first = false; + } + + while (tk.pos < fb.length) + { + u8 ch = fb.data[tk.pos]; + if (ch != 'l' && ch != 'u' && ch != 'U' && ch != 'L' && ch != 'f' && ch != 'F') + { + break; + } + + tk.pos += 1; + } + + t.end = tk.pos; +} + +void +ParseStr(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + + tk.tokens[tk.tk_count].type = TT.DoubleQuote; + + bool ignore = true; + while ((fb.data[tk.pos] != '"' || ignore) && tk.pos < fb.length) + { + tk.pos += 1; + ignore = fb.data[tk.pos] == '\\'; + } + + tk.pos += 1; + tk.tokens[tk.tk_count].end = tk.pos; +} + +void +ParseId(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + Token* t = tk.tokens.ptr + tk.tk_count; + + t.type = TT.Identifier; + t.start = tk.pos; + + while (tk.pos < fb.length) + { + tk.pos += 1; + + u8 ch = fb.data[tk.pos]; + + if (!(ch >= 'a' && ch <= 'z') && !(ch >= 'A' && ch <= 'Z') && ch != '.' && ch != '_' && !(ch >= '0' && ch <= '9')) + { + break; + } + } + + t.end = tk.pos; + + u8 ch = cast(u8)(fb.data[t.start] - cast(u8)(95)); // Index into keywords lookup + if (ch < D_KEYWORDS.length && D_KEYWORDS[ch].length > 0) + { + u8[] id = fb.data[t.start .. t.end]; + foreach(k; D_KEYWORDS[ch]) + { + if (id == k) + { + t.type = TT.Keyword; + break; + } + } + } + + assert(t.start < t.end, "ParseId failure: start is lower or equal to end"); +} + +pragma(inline): bool +CheckEOL(u8 ch) +{ + return ch == 0x0D|| + ch == 0x0A|| + ch == 0x0B|| + ch == 0x0C; +} + +pragma(inline): bool +CheckWhiteSpace(u8 ch) +{ + return ch == ' ' || + ch == '\t'|| + ch == '\n'|| + ch == 0x0D|| + ch == 0x0A|| + ch == 0x0B|| + ch == 0x0C; +} + +void +SkipWhiteSpace(FlatBuffer* fb) +{ + while (fb.tk.pos < fb.length) + { + u8 ch = fb.data[fb.tk.pos]; + + bool white_space = CheckWhiteSpace(ch); + + if (!white_space) + { + break; + } + + fb.tk.pos += 1; + } +} +