diff --git a/dub.json b/dub.json index 973a65c..60c0d30 100644 --- a/dub.json +++ b/dub.json @@ -15,7 +15,7 @@ "libs-windows": [], "preGenerateCommands-linux": ["./build.sh"], "preGenerateCommands-windows": [], - "dflags": ["-Xcc=-mno-sse", "-P-I/usr/include/freetype2"], + "dflags": ["-Xcc=-mno-sse", "-P-I/usr/include/freetype2", "-Jbuild"], "dflags-dmd": ["-P=-DSTBI_NO_SIMD"] }, ] diff --git a/src/DLibs b/src/DLibs index c0d9de9..79d128c 160000 --- a/src/DLibs +++ b/src/DLibs @@ -1 +1 @@ -Subproject commit c0d9de9c4e7fd33e3329745496e601431f6e2970 +Subproject commit 79d128c872e10075e29d19616645d82cb53e3450 diff --git a/src/VulkanRenderer b/src/VulkanRenderer index 16820bd..f3b7536 160000 --- a/src/VulkanRenderer +++ b/src/VulkanRenderer @@ -1 +1 @@ -Subproject commit 16820bdde294d59425fda6e115ae7b2d1e532e51 +Subproject commit f3b7536c8e4edea80d3b7ef3e4265e521a050c03 diff --git a/src/editor/buffer.d b/src/editor/buffer.d index ce20ab5..59dfec2 100644 --- a/src/editor/buffer.d +++ b/src/editor/buffer.d @@ -53,7 +53,7 @@ CreateFlatBuffer(u8[] data) fb.tk = CreateTokenizer(&fb); - Highlight(&fb); + Tokenize(&fb); return fb; } @@ -226,7 +226,7 @@ Replace(FlatBuffer* buffer, u8[] insert, u64 length, u64 pos, u64 delete_length) Insert(buffer, insert, insert.length, pos); } -enum TokenType : u8 +enum TokenStyle : u8 { None = 0, Keyword, @@ -242,75 +242,751 @@ enum TokenType : u8 String, Number, Char, + Exclamation, +} + +alias TS = TokenStyle; + +enum TokenType : u32 +{ + None = 0, + Keyword, + Type, + Identifier, + String, + Char, + Function, + LiteralStr, + Macro, + Exclamation, + At, + Dollar, + Percent, + Caret, + Ampersand, + Asterisk, + LeftBracket, + RightBracket, + VertBar, + LeftBrace, + RightBrace, + LeftSquareBrace, + RightSquareBrace, + Tilde, + BackTick, + Semicolon, + Hash, + Colon, + Dot, + Minus, + Plus, + Equals, + Underscore, + BackSlash, + Slash, + LessThan, + GreaterThan, + Question, + Comma, + Comment, + Number, + SingleQuote, + DoubleQuote, } alias TT = TokenType; +struct Token +{ + TokenType type; + u64 start; + u64 end; +} + struct Tokenizer { - TokenType[] buffer; - u64 pos; - u64 token_end; - u64 length; - TokenType prev_token; - TokenType current_token; - TokenType next_type; + Arena arena; + TokenStyle[] buffer; + Token[] tokens; + u64 tk_count; + u64 pos; } Tokenizer CreateTokenizer(FlatBuffer* buffer) { Tokenizer tokenizer = { - buffer: MAllocArray!(TT)(buffer.data.length), + arena: CreateArena(MB(8)), + buffer: MAllocArray!(TS)(buffer.data.length), }; return tokenizer; } +u8 +PeekNextChar(FlatBuffer* fb) +{ + return fb.tk.pos+1 < fb.length ? fb.data[fb.tk.pos+1] : 0; +} + +const const(u8[])[]['z'-95] D_KEYWORDS = [ + '_'-95: [ + cast(u8[])r"__FILE__", + cast(u8[])r"__FILE_FULL_PATH__", + cast(u8[])r"__FUNCTION__", + cast(u8[])r"__LINE__", + cast(u8[])r"__MODULE__", + cast(u8[])r"__PRETTY_FUNCTION__", + cast(u8[])r"__gshared", + cast(u8[])r"__parameters", + cast(u8[])r"__rvalue", + cast(u8[])r"__traits", + cast(u8[])r"__vector", + ], + 'a'-95: [ + cast(u8[])r"abstract", + cast(u8[])r"alias", + cast(u8[])r"align", + cast(u8[])r"asm", + cast(u8[])r"assert", + cast(u8[])r"auto", + ], + 'b'-95: [ + cast(u8[])r"bool", + cast(u8[])r"break", + cast(u8[])r"byte", + ], + 'c'-95: [ + cast(u8[])r"case", + cast(u8[])r"cast", + cast(u8[])r"catch", + cast(u8[])r"char", + cast(u8[])r"class", + cast(u8[])r"const", + cast(u8[])r"continue", + ], + 'd'-95: [ + cast(u8[])r"dchar", + cast(u8[])r"debug", + cast(u8[])r"default", + cast(u8[])r"delegate", + cast(u8[])r"deprecated", + cast(u8[])r"do", + cast(u8[])r"double", + ], + 'e'-95: [ + cast(u8[])r"else", + cast(u8[])r"enum", + cast(u8[])r"export", + cast(u8[])r"extern", + ], + 'f'-95: [ + cast(u8[])r"false", + cast(u8[])r"final", + cast(u8[])r"finally", + cast(u8[])r"float", + cast(u8[])r"for", + cast(u8[])r"foreach", + cast(u8[])r"foreach_reverse", + cast(u8[])r"function", + ], + 'g'-95: [ + cast(u8[])r"goto", + ], + 'i'-95: [ + cast(u8[])r"if", + cast(u8[])r"immutable", + cast(u8[])r"import", + cast(u8[])r"in", + cast(u8[])r"inout", + cast(u8[])r"int", + cast(u8[])r"interface", + cast(u8[])r"invariant", + cast(u8[])r"is", + ], + 'l'-95: [ + cast(u8[])r"lazy", + cast(u8[])r"long", + ], + 'm'-95: [ + cast(u8[])r"mixin", + cast(u8[])r"module", + ], + 'n'-95: [ + cast(u8[])r"new", + cast(u8[])r"nothrow", + cast(u8[])r"null", + ], + 'o'-95: [ + cast(u8[])r"out", + cast(u8[])r"override", + ], + 'p'-95: [ + cast(u8[])r"package", + cast(u8[])r"pragma", + cast(u8[])r"private", + cast(u8[])r"protected", + cast(u8[])r"public", + cast(u8[])r"pure", + ], + 'r'-95: [ + cast(u8[])r"real", + cast(u8[])r"ref", + cast(u8[])r"return", + ], + 's'-95: [ + cast(u8[])r"scope", + cast(u8[])r"shared", + cast(u8[])r"short", + cast(u8[])r"static", + cast(u8[])r"struct", + cast(u8[])r"super", + cast(u8[])r"switch", + cast(u8[])r"synchronized", + ], + 't'-95: [ + cast(u8[])r"template", + cast(u8[])r"this", + cast(u8[])r"throw", + cast(u8[])r"true", + cast(u8[])r"try", + cast(u8[])r"typeid", + cast(u8[])r"typeof", + ], + 'u'-95: [ + cast(u8[])r"ubyte", + cast(u8[])r"uint", + cast(u8[])r"ulong", + cast(u8[])r"union", + cast(u8[])r"unittest", + cast(u8[])r"ushort", + ], + 'v'-95: [ + cast(u8[])r"version", + cast(u8[])r"void", + ], + 'w'-95: [ + cast(u8[])r"wchar", + cast(u8[])r"while", + cast(u8[])r"with", + ], +]; + +void +Tokenize(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + + u64 semi_count; + foreach(i; 0 .. fb.length) + { + if (fb.data.ptr[i] == ';') + { + semi_count += 1; + } + } + + Reset(&tk.arena); + + tk.tokens = AllocArray!(Token)(&tk.arena, fb.length); + tk.tk_count = 0; + + bool _macro = false; + Token* prev = null; + while (true) + { + Token* t = NextToken(fb); + if (t == null) break; + + switch(t.type) + { + case TT.Keyword: + { + tk.buffer[t.start .. t.end] = TS.Keyword; + } break; + case TT.Number: + { + tk.buffer[t.start .. t.end] = TS.Number; + } break; + case TT.Comment: + { + tk.buffer[t.start .. t.end] = TS.Comment; + } break; + case TT.DoubleQuote: + case TT.BackTick: + { + tk.buffer[t.start .. t.end] = TS.String; + } break; + case TT.SingleQuote: + { + tk.buffer[t.start .. t.end] = TS.Char; + } break; + case TT.Identifier: + { + tk.buffer[t.start .. t.end] = TS.Identifier; + } break; + case TT.LeftBracket: + { + if (prev != null && _macro && prev.type == TT.Exclamation) + { + + } + } goto case TT.Comma; + case TT.LeftBrace: + { + if (prev) + { + if (prev.type == TT.RightBracket) + { + i64 i = tk.tk_count - 3; + Token* pt = null; + bool is_macro = false; + i64 brackets = 1; + while (i > 0) + { + pt = tk.tokens.ptr + i; + i -= 1; + + if (pt.type == TT.Keyword && brackets == 0) break; + + if (pt.type == TT.LeftBracket) + { + brackets -= 1; + continue; + } + + if (pt.type == TT.RightBracket) + { + brackets += 1; + is_macro = true; + break; + } + + if (pt.type == TT.Identifier && brackets == 0) + { + pt.type = TT.Function; + tk.buffer[pt.start .. pt.end] = TS.Function; + + brackets = 0; + Token* ppt = null; + while (i < tk.tk_count - 2) + { + ppt = pt; + i += 1; + pt = tk.tokens.ptr + i; + + bool id_keyword = pt.type == TT.Identifier || pt.type == TT.Keyword; + + if ( is_macro && brackets == 4) break; + if (!is_macro && brackets == 2) break; + + if (is_macro && brackets < 2 && id_keyword) + { + pt.type = TT.Type; + tk.buffer[pt.start .. pt.end] = TS.Type; + continue; + } + + if (!is_macro && brackets < 2 && id_keyword && (ppt.type == TT.LeftBracket || ppt.type == TT.Comma)) + { + pt.type = TT.Type; + tk.buffer[pt.start .. pt.end] = TS.Type; + continue; + } + + if (pt.type == TT.LeftBracket || pt.type == TT.RightBracket) + { + brackets += 1; + continue; + } + } + + break; + } + } + } + } + } goto case TT.Comma; + case TT.Semicolon: + case TT.RightBracket: + case TT.RightBrace: + case TT.LeftSquareBrace: + case TT.RightSquareBrace: + case TT.Colon: + case TT.Comma: + { + tk.buffer[t.start .. t.end] = TS.Bracket; + } break; + case TT.Exclamation: + { + _macro = true; + } goto case TT.Equals; + case TT.Tilde: + case TT.Question: + case TT.LessThan: + case TT.GreaterThan: + case TT.Dollar: + case TT.Percent: + case TT.Caret: + case TT.Ampersand: + case TT.Asterisk: + case TT.VertBar: + case TT.Plus: + case TT.Minus: + case TT.Slash: + case TT.Equals: + { + tk.buffer[t.start .. t.end] = TS.Op; + } break; + default: + tk.buffer[t.start .. t.end] = TS.Keyword; + } + + prev = t; + } +} + +Token* +NextToken(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + + SkipWhiteSpace(fb); + + u8 ch = fb.data[tk.pos]; + + Token* t = null; + if (tk.pos < fb.length) + { + t = tk.tokens.ptr + tk.tk_count; + t.start = tk.pos; + + bool ascii = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'); + + if (ch == 'r') + { + u8 next = PeekNextChar(fb); + if (next == '"') + { + tk.pos += 1; + ParseStr(fb); + } + else + { + ParseId(fb); + } + } + else if (ch >= '0' && ch <= '9') + { + ParseNum(fb); + } + else if (ch == 'P' || ch == 'E' || ch == 'e' || ch == 'p') + { + u8 next = PeekNextChar(fb); + if (next == '-' || next == '+' || (next >= '0' && next <= '9')) + { + ParseNum(fb); + } + else + { + ParseId(fb); + } + } + else if (ascii || ch == '_') + { + ParseId(fb); + } + else + { + switch (ch) + { + case '$': t.type = TT.Dollar; goto case u8.max; + case '(': t.type = TT.LeftBracket; goto case u8.max; + case ')': t.type = TT.RightBracket; goto case u8.max; + case '[': t.type = TT.LeftSquareBrace; goto case u8.max; + case ']': t.type = TT.RightSquareBrace; goto case u8.max; + case '{': t.type = TT.LeftBrace; goto case u8.max; + case '}': t.type = TT.RightBrace; goto case u8.max; + case '|': t.type = TT.VertBar; goto case u8.max; + case ';': t.type = TT.Semicolon; goto case u8.max; + case ':': t.type = TT.Colon; goto case u8.max; + case '?': t.type = TT.Question; goto case u8.max; + case ',': t.type = TT.Comma; goto case u8.max; + case '#': t.type = TT.Hash; goto case u8.max; + case '~': t.type = TT.Tilde; goto case u8.max; + case '%': t.type = TT.Percent; goto case 251; + case '^': t.type = TT.Caret; goto case 251; + case '&': t.type = TT.Ampersand; goto case 251; + case '*': t.type = TT.Asterisk; goto case 251; + case '-': t.type = TT.Minus; goto case 251; + case '=': t.type = TT.Equals; goto case 251; + case '+': t.type = TT.Plus; goto case 251; + case '<': t.type = TT.LessThan; goto case 251; + case '>': t.type = TT.GreaterThan; goto case 251; + + case '`': + { + t.type = TT.BackTick; + tk.pos += 1; + while (fb.data[tk.pos] != '`' && tk.pos < fb.length) + { + tk.pos += 1; + } + t.end = tk.pos+1; + } break; + case '\'': + { + t.type = TT.SingleQuote; + + bool ignore = true; + while ((fb.data[tk.pos] != '\'' || ignore) && tk.pos < fb.length) + { + tk.pos += 1; + ignore = fb.data[tk.pos] == '\\'; + } + + tk.pos += 1; + t.end = tk.pos; + } break; + case '"': + ParseStr(fb); + break; + case '!': + t.type = TT.Exclamation; + goto case 251; + case '@': + { + t.type = TT.At; + while (tk.pos < fb.length) + { + tk.pos += 1; + u8 c = fb.data[tk.pos]; + + if (CheckWhiteSpace(c)) break; + if (!(c >= 'a' && c <= 'z') && !(c >= 'A' && c <= 'Z')) break; + } + + t.end = tk.pos; + } break; + case '.': + { + t.type = TT.Dot; + while (tk.pos < fb.length) + { + tk.pos += 1; + if (tk.pos != '.') break; + } + t.end = tk.pos; + } break; + case '/': + { + t.type = TT.Slash; + u8 next = PeekNextChar(fb); + if (next == '/') + { + tk.pos += 1; + t.type = TT.Comment; + + while (tk.pos < fb.length) + { + tk.pos += 1; + + if (CheckEOL(fb.data[tk.pos])) + { + break; + } + } + + t.end = tk.pos+1; + } + else + { + goto case u8.max; + } + } break; + case 251: + { + u8 next = PeekNextChar(fb); + if (next == '=') + { + tk.pos += 1; + } + + t.end = tk.pos+1; + tk.pos += 1; + } break; + case u8.max: + { + t.end = tk.pos+1; + tk.pos += 1; + } break; + default: + t.type = TT.None; + t.end = tk.pos+1; + tk.pos += 1; + } + } + + tk.tk_count += 1; + } + + return t; +} + +void +ParseNum(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + + Token* t = tk.tokens.ptr + tk.tk_count; + t.type = TT.Number; + t.start = tk.pos; + + bool first = true; + while (tk.pos < fb.length) + { + tk.pos += 1; + + u8 ch = fb.data[tk.pos]; + + if (ch != '_' && ch != '.' && !(ch >= '0' && ch <= '9') && ch != 'x' && ch != 'X' && !((ch == '-' || ch == '+') && first)) + { + break; + } + + first = false; + } + + while (tk.pos < fb.length) + { + u8 ch = fb.data[tk.pos]; + if (ch != 'l' && ch != 'u' && ch != 'U' && ch != 'L' && ch != 'f' && ch != 'F') + { + break; + } + + tk.pos += 1; + } + + t.end = tk.pos; +} + +void +ParseStr(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + + tk.tokens[tk.tk_count].type = TT.DoubleQuote; + + bool ignore = true; + while ((fb.data[tk.pos] != '"' || ignore) && tk.pos < fb.length) + { + tk.pos += 1; + ignore = fb.data[tk.pos] == '\\'; + } + + tk.pos += 1; + tk.tokens[tk.tk_count].end = tk.pos; +} + +void +ParseId(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + Token* t = tk.tokens.ptr + tk.tk_count; + + t.type = TT.Identifier; + t.start = tk.pos; + + while (tk.pos < fb.length) + { + tk.pos += 1; + + u8 ch = fb.data[tk.pos]; + + if (!(ch >= 'a' && ch <= 'z') && !(ch >= 'A' && ch <= 'Z') && ch != '.' && ch != '_' && !(ch >= '0' && ch <= '9')) + { + break; + } + } + + t.end = tk.pos; + + u8 ch = cast(u8)(fb.data[t.start] - cast(u8)(95)); // Index into keywords lookup + if (ch < D_KEYWORDS.length && D_KEYWORDS[ch].length > 0) + { + u8[] id = fb.data[t.start .. t.end]; + foreach(k; D_KEYWORDS[ch]) + { + if (id == k) + { + t.type = TT.Keyword; + break; + } + } + } + + assert(t.start < t.end, "ParseId failure: start is lower or equal to end"); +} + +pragma(inline): bool +CheckEOL(u8 ch) +{ + return ch == 0x0D|| + ch == 0x0A|| + ch == 0x0B|| + ch == 0x0C; +} + pragma(inline): bool CheckWhiteSpace(u8 ch) { - return ch == ' ' || - ch == '\t' || - ch == '\n' || - ch == EOF; + return ch == ' ' || + ch == '\t'|| + ch == '\n'|| + ch == 0x0D|| + ch == 0x0A|| + ch == 0x0B|| + ch == 0x0C; } -pragma(inline): bool -CheckDelimiter(u8 ch) +void +SkipWhiteSpace(FlatBuffer* fb) { - return ch == ';' || - ch == ',' || - ch == '[' || - ch == '(' || - ch == '{' || - ch == '}' || - ch == ']' || - ch == ')' || - ch == '>' || - ch == '<'; -} - -pragma(inline): bool -CheckString(u8 ch) -{ - return ch == '\'' || - ch == '"' || - ch == '`'; + while (fb.tk.pos < fb.length) + { + u8 ch = fb.data[fb.tk.pos]; + + bool white_space = CheckWhiteSpace(ch); + + if (!white_space) + { + break; + } + + fb.tk.pos += 1; + } } +/* bool Advance(FlatBuffer* fb) { Tokenizer* tk = &fb.tk; - tk.pos = tk.token_end; - tk.next_type = TT.None; + tk.pos = tk.pos_end; + tk.next_type = TS.None; u8[] buf = fb.data; + tk.pos_next = tk.pos; + tk.pos_end_next = tk.pos_end; + + tk.pos = tk.pos == -1 ? 0 : tk.pos; + tk.pos_end = tk.pos_end == -1 ? 0 : tk.pos_end; + bool result = false; bool started = false; bool str_started = false; - for(u64 i = tk.pos; i < buf.length; i += 1) + for(i64 i = tk.pos; i < buf.length; i += 1) { u8 ch = buf.ptr[i]; bool space = CheckWhiteSpace(ch); @@ -320,7 +996,7 @@ Advance(FlatBuffer* fb) if (!started && delim) { tk.pos = i; - tk.token_end = i+1; + tk.pos_end = i+1; result = true; break; } @@ -336,15 +1012,28 @@ Advance(FlatBuffer* fb) if (started && (delim || space) && !str_started) { result = true; - tk.token_end = i; - tk.next_type = ch == '(' ? TT.Bracket : TT.None; + tk.pos_end = i; + + if (ch == EOF) + { + Logf("final token"); + tk.final_token = true; + } + + switch(ch) + { + case '(': tk.next_type = TS.Bracket; break; + case '!': tk.next_type = TS.Exclamation; break; + default: break; + } + break; } if (started && str_started && str) { result = true; - tk.token_end = i+1; + tk.pos_end = i+1; break; } } @@ -370,144 +1059,144 @@ Highlight(FlatBuffer* fb) { const const(u8[])[]['z'-95] keywords = [ '_'-95: [ - mixin(StrToU8("__FILE__")), - mixin(StrToU8("__FILE_FULL_PATH__")), - mixin(StrToU8("__FUNCTION__")), - mixin(StrToU8("__LINE__")), - mixin(StrToU8("__MODULE__")), - mixin(StrToU8("__PRETTY_FUNCTION__")), - mixin(StrToU8("__gshared")), - mixin(StrToU8("__parameters")), - mixin(StrToU8("__rvalue")), - mixin(StrToU8("__traits")), - mixin(StrToU8("__vector")), + cast(u8[])r"__FILE__", + cast(u8[])r"__FILE_FULL_PATH__", + cast(u8[])r"__FUNCTION__", + cast(u8[])r"__LINE__", + cast(u8[])r"__MODULE__", + cast(u8[])r"__PRETTY_FUNCTION__", + cast(u8[])r"__gshared", + cast(u8[])r"__parameters", + cast(u8[])r"__rvalue", + cast(u8[])r"__traits", + cast(u8[])r"__vector", ], 'a'-95: [ - mixin(StrToU8("abstract")), - mixin(StrToU8("alias")), - mixin(StrToU8("align")), - mixin(StrToU8("asm")), - mixin(StrToU8("assert")), - mixin(StrToU8("auto")), + cast(u8[])r"abstract", + cast(u8[])r"alias", + cast(u8[])r"align", + cast(u8[])r"asm", + cast(u8[])r"assert", + cast(u8[])r"auto", ], 'b'-95: [ - mixin(StrToU8("bool")), - mixin(StrToU8("break")), - mixin(StrToU8("byte")), + cast(u8[])r"bool", + cast(u8[])r"break", + cast(u8[])r"byte", ], 'c'-95: [ - mixin(StrToU8("case")), - mixin(StrToU8("cast")), - mixin(StrToU8("catch")), - mixin(StrToU8("char")), - mixin(StrToU8("class")), - mixin(StrToU8("const")), - mixin(StrToU8("continue")), + cast(u8[])r"case", + cast(u8[])r"cast", + cast(u8[])r"catch", + cast(u8[])r"char", + cast(u8[])r"class", + cast(u8[])r"const", + cast(u8[])r"continue", ], 'd'-95: [ - mixin(StrToU8("dchar")), - mixin(StrToU8("debug")), - mixin(StrToU8("default")), - mixin(StrToU8("delegate")), - mixin(StrToU8("deprecated")), - mixin(StrToU8("do")), - mixin(StrToU8("double")), + cast(u8[])r"dchar", + cast(u8[])r"debug", + cast(u8[])r"default", + cast(u8[])r"delegate", + cast(u8[])r"deprecated", + cast(u8[])r"do", + cast(u8[])r"double", ], 'e'-95: [ - mixin(StrToU8("else")), - mixin(StrToU8("enum")), - mixin(StrToU8("export")), - mixin(StrToU8("extern")), + cast(u8[])r"else", + cast(u8[])r"enum", + cast(u8[])r"export", + cast(u8[])r"extern", ], 'f'-95: [ - mixin(StrToU8("false")), - mixin(StrToU8("final")), - mixin(StrToU8("finally")), - mixin(StrToU8("float")), - mixin(StrToU8("for")), - mixin(StrToU8("foreach")), - mixin(StrToU8("foreach_reverse")), - mixin(StrToU8("function")), + cast(u8[])r"false", + cast(u8[])r"final", + cast(u8[])r"finally", + cast(u8[])r"float", + cast(u8[])r"for", + cast(u8[])r"foreach", + cast(u8[])r"foreach_reverse", + cast(u8[])r"function", ], 'g'-95: [ - mixin(StrToU8("goto")), + cast(u8[])r"goto", ], 'i'-95: [ - mixin(StrToU8("if")), - mixin(StrToU8("immutable")), - mixin(StrToU8("import")), - mixin(StrToU8("in")), - mixin(StrToU8("inout")), - mixin(StrToU8("int")), - mixin(StrToU8("interface")), - mixin(StrToU8("invariant")), - mixin(StrToU8("is")), + cast(u8[])r"if", + cast(u8[])r"immutable", + cast(u8[])r"import", + cast(u8[])r"in", + cast(u8[])r"inout", + cast(u8[])r"int", + cast(u8[])r"interface", + cast(u8[])r"invariant", + cast(u8[])r"is", ], 'l'-95: [ - mixin(StrToU8("lazy")), - mixin(StrToU8("long")), + cast(u8[])r"lazy", + cast(u8[])r"long", ], 'm'-95: [ - mixin(StrToU8("mixin")), - mixin(StrToU8("module")), + cast(u8[])r"mixin", + cast(u8[])r"module", ], 'n'-95: [ - mixin(StrToU8("new")), - mixin(StrToU8("nothrow")), - mixin(StrToU8("null")), + cast(u8[])r"new", + cast(u8[])r"nothrow", + cast(u8[])r"null", ], 'o'-95: [ - mixin(StrToU8("out")), - mixin(StrToU8("override")), + cast(u8[])r"out", + cast(u8[])r"override", ], 'p'-95: [ - mixin(StrToU8("package")), - mixin(StrToU8("pragma")), - mixin(StrToU8("private")), - mixin(StrToU8("protected")), - mixin(StrToU8("public")), - mixin(StrToU8("pure")), + cast(u8[])r"package", + cast(u8[])r"pragma", + cast(u8[])r"private", + cast(u8[])r"protected", + cast(u8[])r"public", + cast(u8[])r"pure", ], 'r'-95: [ - mixin(StrToU8("real")), - mixin(StrToU8("ref")), - mixin(StrToU8("return")), + cast(u8[])r"real", + cast(u8[])r"ref", + cast(u8[])r"return", ], 's'-95: [ - mixin(StrToU8("scope")), - mixin(StrToU8("shared")), - mixin(StrToU8("short")), - mixin(StrToU8("static")), - mixin(StrToU8("struct")), - mixin(StrToU8("super")), - mixin(StrToU8("switch")), - mixin(StrToU8("synchronized")), + cast(u8[])r"scope", + cast(u8[])r"shared", + cast(u8[])r"short", + cast(u8[])r"static", + cast(u8[])r"struct", + cast(u8[])r"super", + cast(u8[])r"switch", + cast(u8[])r"synchronized", ], 't'-95: [ - mixin(StrToU8("template")), - mixin(StrToU8("this")), - mixin(StrToU8("throw")), - mixin(StrToU8("true")), - mixin(StrToU8("try")), - mixin(StrToU8("typeid")), - mixin(StrToU8("typeof")), + cast(u8[])r"template", + cast(u8[])r"this", + cast(u8[])r"throw", + cast(u8[])r"true", + cast(u8[])r"try", + cast(u8[])r"typeid", + cast(u8[])r"typeof", ], 'u'-95: [ - mixin(StrToU8("ubyte")), - mixin(StrToU8("uint")), - mixin(StrToU8("ulong")), - mixin(StrToU8("union")), - mixin(StrToU8("unittest")), - mixin(StrToU8("ushort")), + cast(u8[])r"ubyte", + cast(u8[])r"uint", + cast(u8[])r"ulong", + cast(u8[])r"union", + cast(u8[])r"unittest", + cast(u8[])r"ushort", ], 'v'-95: [ - mixin(StrToU8("version")), - mixin(StrToU8("void")), + cast(u8[])r"version", + cast(u8[])r"void", ], 'w'-95: [ - mixin(StrToU8("wchar")), - mixin(StrToU8("while")), - mixin(StrToU8("with")), + cast(u8[])r"wchar", + cast(u8[])r"while", + cast(u8[])r"with", ], ]; @@ -515,37 +1204,49 @@ Highlight(FlatBuffer* fb) u8[] token; u8[] prev_token; + if (tk.final_token) + { + goto InnerTokenStream; + } + TokenStream: while (Advance(fb)) { - token = fb.data[tk.pos .. tk.token_end]; +InnerTokenStream: + token = fb.data[tk.pos .. tk.pos_end]; scope(exit) prev_token = token; if (token.length == 0) break; if (token.length == 1 && CheckDelimiter(token.ptr[0])) { - tk.buffer[tk.pos] = TT.Bracket; + tk.buffer[tk.pos] = TS.Bracket; continue TokenStream; } if (CheckString(token[0]) && CheckString(token[token.length-1]) && token[0] == token[token.length-1]) { - tk.buffer[tk.pos .. tk.token_end] = TT.String; + tk.buffer[tk.pos .. tk.pos_end] = TS.String; continue TokenStream; } if (token[0] >= '0' && token[0] <= '9') { - tk.buffer[tk.pos .. tk.token_end] = TT.Number; + tk.buffer[tk.pos .. tk.pos_end] = TS.Number; continue TokenStream; } if ((token[0] >= 'a' && token[0] <= 'z') || (token[0] >= 'A' && token[0] <= 'Z') || token[0] == '_') { - if (prev_token == mixin(StrToU8("import"))) + if (prev_token == r"import") { - tk.buffer[tk.pos .. tk.token_end] = TT.ImportTarget; + tk.buffer[tk.pos .. tk.pos_end] = TS.ImportTarget; + continue TokenStream; + } + + if (tk.next_type == TS.Exclamation) + { + tk.buffer[tk.pos .. tk.pos_end] = TS.Macro; continue TokenStream; } @@ -558,18 +1259,20 @@ TokenStream: { if (token == key) { - tk.buffer[tk.pos .. tk.token_end] = TT.Keyword; + tk.buffer[tk.pos .. tk.pos_end] = TS.Keyword; continue TokenStream; } } } - tk.buffer[tk.pos .. tk.token_end] = (tk.next_type == TT.Bracket) ? TT.Function : TT.Identifier; + tk.buffer[tk.pos .. tk.pos_end] = (tk.next_type == TS.Bracket) ? TS.Function : TS.Identifier; continue TokenStream; } } } +*/ + unittest { import std.stdio; diff --git a/src/editor/editor.d b/src/editor/editor.d index 1fe25ce..94ecf6e 100644 --- a/src/editor/editor.d +++ b/src/editor/editor.d @@ -212,51 +212,55 @@ DrawText(Editor* ed, f32 x, f32 y, f32 px, string str) void DrawBuffer(Editor* ed, f32 x, f32 y, f32 px, FlatBuffer* fb) { - const Vec4[TT.max] cols = [ - TT.None: Vec4(1.0, 1.0, 1.0, 1.0), - TT.ImportTarget: Vec4(0.4, 0.7, 0.2, 1.0), - TT.Keyword: Vec4(1.0, 0.0, 0.0, 1.0), - TT.Bracket: Vec4(0.3, 0.3, 0.3, 1.0), - TT.String: Vec4(0.2, 0.6, 0.2, 1.0), - TT.Number: Vec4(0.2, 0.3, 0.8, 1.0), - TT.Function: Vec4(0.4, 0.3, 0.7, 1.0), - TT.Identifier: Vec4(0.8, 0.4, 0.5, 1.0), + const Vec4[TS.max] cols = [ + TS.None: Vec4(1.0, 1.0, 1.0, 1.0), + TS.ImportTarget: Vec4(0.4, 0.7, 0.2, 1.0), + TS.Keyword: Vec4(1.0, 0.0, 0.0, 1.0), + TS.Bracket: Vec4(0.3, 0.3, 0.3, 1.0), + TS.String: Vec4(0.2, 0.6, 0.2, 1.0), + TS.Char: Vec4(0.3, 0.5, 0.5, 1.0), + TS.Number: Vec4(0.2, 0.3, 0.8, 1.0), + TS.Function: Vec4(0.4, 0.3, 0.7, 1.0), + TS.Identifier: Vec4(0.8, 0.4, 0.5, 1.0), + TS.Macro: Vec4(0.1, 0.3, 0.3, 1.0), + TS.Op: Vec4(0.5, 0.8, 0.2, 1.0), + TS.Comment: Vec4(0.3, 0.3, 0.3, 1.0), + TS.Type: Vec4(1.0, 1.0, 0.5, 1.0), ]; u32 tab_count = 2; f32 x_pos = x; - f32 y_pos = y; + f32 y_pos = y + px; f32 scale = px / ed.atlas_buf.atlas.size; foreach(i; 0 .. fb.length) { u8 ch = fb.data[i]; - foreach(glyph; ed.atlas_buf.atlas.glyphs) + if (ch > 0 && ch < ed.atlas_buf.atlas.glyphs.length) { - if (ch == glyph.ch) + Glyph* g = ed.atlas_buf.atlas.glyphs.ptr + ch; + + if (g.advance > 0.0) { if (ch == '\t') { - Glyph g = glyph; g.atlas_left = g.atlas_right = 0.0; foreach(j; 0 .. tab_count) { - DrawGlyph(ed, &g, scale, &x_pos, y_pos); + DrawGlyph(ed, g, scale, &x_pos, y_pos); } } else if (ch == '\n') { - Glyph g = glyph; g.atlas_left = g.atlas_right = 0.0; - DrawGlyph(ed, &g, scale, &x_pos, y_pos); + DrawGlyph(ed, g, scale, &x_pos, y_pos); y_pos += px; x_pos = 0.0; } else { - DrawGlyph(ed, &glyph, scale, &x_pos, y_pos, cols[fb.tk.buffer[i]]); - break; + DrawGlyph(ed, g, scale, &x_pos, y_pos, cols[fb.tk.buffer[i]]); } } } diff --git a/test/files/syntax_highlighting.d b/test/files/syntax_highlighting.d new file mode 100644 index 0000000..d8a553e --- /dev/null +++ b/test/files/syntax_highlighting.d @@ -0,0 +1,70 @@ +import std.stdio; + +@nogc: bool +Test() +{ + int x = 5; + int y = 10; + + x = x + y; + x = x - y; + x = x / y; + x = x * y; + x = x % y; + + x += y; + x -= y; + x /= y; + x *= y; + x %= y; + + x = x & y; + x = x | y; + x = x ^ y; + x = ~x; + x ~= x; + + // Commented here + if (x > 1L && y < 2u || x <= 3U && y >= 4Lu || x == 5LU && x != 6uL || x == 7UL) + { + return true; + } + + float z = 10.5F; + float w = 11.25f; + real r = 200.55L; + + z = e+102.5; + z = e-00.25; + + int hex = 0x555; + int hex2 = 0X555; + + int hexp = p+0X5555; + hexp = p-0x555; + + string str_literal = r"Teststring"; + string str = "test string"; + char ch = 'c'; + + u8[] array = [ + 1, 2, 3, 4, 5 + ]; + + array[5] = 22; + + int value = WithParameters(z, str); + + return false; +} + +int WithParameters(float x, string y) +{ + assert(y.length > 0, "String must not be empty"); + return cast(int)(x) - y[0]; +} + +T MacroFunc(T)(T x) +{ + return x; +}