diff --git a/src/editor/buffer.d b/src/editor/buffer.d index 3477f9f..ce20ab5 100644 --- a/src/editor/buffer.d +++ b/src/editor/buffer.d @@ -254,6 +254,7 @@ struct Tokenizer u64 length; TokenType prev_token; TokenType current_token; + TokenType next_type; } Tokenizer @@ -266,28 +267,84 @@ CreateTokenizer(FlatBuffer* buffer) return tokenizer; } +pragma(inline): bool +CheckWhiteSpace(u8 ch) +{ + return ch == ' ' || + ch == '\t' || + ch == '\n' || + ch == EOF; +} + +pragma(inline): bool +CheckDelimiter(u8 ch) +{ + return ch == ';' || + ch == ',' || + ch == '[' || + ch == '(' || + ch == '{' || + ch == '}' || + ch == ']' || + ch == ')' || + ch == '>' || + ch == '<'; +} + +pragma(inline): bool +CheckString(u8 ch) +{ + return ch == '\'' || + ch == '"' || + ch == '`'; +} + bool Advance(FlatBuffer* fb) { Tokenizer* tk = &fb.tk; tk.pos = tk.token_end; + tk.next_type = TT.None; u8[] buf = fb.data; bool result = false; bool started = false; + bool str_started = false; for(u64 i = tk.pos; i < buf.length; i += 1) { - if (!started && buf.ptr[i] != ' ' && buf.ptr[i] != '\t' && buf.ptr[i] != '\n' && buf.ptr[i] != EOF) + u8 ch = buf.ptr[i]; + bool space = CheckWhiteSpace(ch); + bool delim = CheckDelimiter(ch); + bool str = CheckString(ch); + + if (!started && delim) + { + tk.pos = i; + tk.token_end = i+1; + result = true; + break; + } + + if (!started && !space) { tk.pos = i; started = true; + str_started = str; continue; } - if (started && (buf.ptr[i] == ' ' || buf.ptr[i] == '\t' || buf.ptr[i] == '\n' || buf.ptr[i] == EOF)) + if (started && (delim || space) && !str_started) { result = true; tk.token_end = i; + tk.next_type = ch == '(' ? TT.Bracket : TT.None; + break; + } + + if (started && str_started && str) + { + result = true; + tk.token_end = i+1; break; } } @@ -455,14 +512,43 @@ Highlight(FlatBuffer* fb) ]; Tokenizer* tk = &fb.tk; + u8[] token; + u8[] prev_token; TokenStream: while (Advance(fb)) { - u8[] token = fb.data[tk.pos .. tk.token_end]; + token = fb.data[tk.pos .. tk.token_end]; + scope(exit) prev_token = token; + + if (token.length == 0) break; + + if (token.length == 1 && CheckDelimiter(token.ptr[0])) + { + tk.buffer[tk.pos] = TT.Bracket; + continue TokenStream; + } + + if (CheckString(token[0]) && CheckString(token[token.length-1]) && token[0] == token[token.length-1]) + { + tk.buffer[tk.pos .. tk.token_end] = TT.String; + continue TokenStream; + } + + if (token[0] >= '0' && token[0] <= '9') + { + tk.buffer[tk.pos .. tk.token_end] = TT.Number; + continue TokenStream; + } if ((token[0] >= 'a' && token[0] <= 'z') || (token[0] >= 'A' && token[0] <= 'Z') || token[0] == '_') { + if (prev_token == mixin(StrToU8("import"))) + { + tk.buffer[tk.pos .. tk.token_end] = TT.ImportTarget; + continue TokenStream; + } + u8 index = cast(u8)(token[0]-cast(u8)95); if (index > 0 && index < keywords.length) { @@ -477,6 +563,9 @@ TokenStream: } } } + + tk.buffer[tk.pos .. tk.token_end] = (tk.next_type == TT.Bracket) ? TT.Function : TT.Identifier; + continue TokenStream; } } } diff --git a/src/editor/editor.d b/src/editor/editor.d index 873b8eb..1fe25ce 100644 --- a/src/editor/editor.d +++ b/src/editor/editor.d @@ -214,7 +214,13 @@ DrawBuffer(Editor* ed, f32 x, f32 y, f32 px, FlatBuffer* fb) { const Vec4[TT.max] cols = [ TT.None: Vec4(1.0, 1.0, 1.0, 1.0), + TT.ImportTarget: Vec4(0.4, 0.7, 0.2, 1.0), TT.Keyword: Vec4(1.0, 0.0, 0.0, 1.0), + TT.Bracket: Vec4(0.3, 0.3, 0.3, 1.0), + TT.String: Vec4(0.2, 0.6, 0.2, 1.0), + TT.Number: Vec4(0.2, 0.3, 0.8, 1.0), + TT.Function: Vec4(0.4, 0.3, 0.7, 1.0), + TT.Identifier: Vec4(0.8, 0.4, 0.5, 1.0), ]; u32 tab_count = 2;