From 3b67ffd06687d1db4ac1ba1d84c556e30d89a791 Mon Sep 17 00:00:00 2001 From: Matthew Date: Tue, 19 Aug 2025 06:09:15 +1000 Subject: [PATCH] starting work on syntax highlighting --- src/editor/buffer.d | 267 +++++++++++++++++++++++++++++++++++++++++++- src/editor/editor.d | 67 ++++++++--- 2 files changed, 319 insertions(+), 15 deletions(-) diff --git a/src/editor/buffer.d b/src/editor/buffer.d index 0c26f68..3477f9f 100644 --- a/src/editor/buffer.d +++ b/src/editor/buffer.d @@ -2,9 +2,11 @@ import aliases; import math; import alloc; import util; +import core.stdc.stdio : EOF; struct FlatBuffer { + Tokenizer tk; Arena arena; u8[] data; u64 length; @@ -42,12 +44,18 @@ CreateFlatBuffer(u8[] data) } } - return FlatBuffer( + FlatBuffer fb = { arena: CreateArena(MB(1)), data: buf, length: cast(u64)data.length, line_count: line_count, - ); + }; + + fb.tk = CreateTokenizer(&fb); + + Highlight(&fb); + + return fb; } LineBuffers @@ -218,6 +226,261 @@ Replace(FlatBuffer* buffer, u8[] insert, u64 length, u64 pos, u64 delete_length) Insert(buffer, insert, insert.length, pos); } +enum TokenType : u8 +{ + None = 0, + Keyword, + Import, + ImportTarget, + Type, + Identifier, + Op, + Bracket, + Function, + Macro, + Comment, + String, + Number, + Char, +} + +alias TT = TokenType; + +struct Tokenizer +{ + TokenType[] buffer; + u64 pos; + u64 token_end; + u64 length; + TokenType prev_token; + TokenType current_token; +} + +Tokenizer +CreateTokenizer(FlatBuffer* buffer) +{ + Tokenizer tokenizer = { + buffer: MAllocArray!(TT)(buffer.data.length), + }; + + return tokenizer; +} + +bool +Advance(FlatBuffer* fb) +{ + Tokenizer* tk = &fb.tk; + tk.pos = tk.token_end; + u8[] buf = fb.data; + + bool result = false; + bool started = false; + for(u64 i = tk.pos; i < buf.length; i += 1) + { + if (!started && buf.ptr[i] != ' ' && buf.ptr[i] != '\t' && buf.ptr[i] != '\n' && buf.ptr[i] != EOF) + { + tk.pos = i; + started = true; + continue; + } + + if (started && (buf.ptr[i] == ' ' || buf.ptr[i] == '\t' || buf.ptr[i] == '\n' || buf.ptr[i] == EOF)) + { + result = true; + tk.token_end = i; + break; + } + } + + return result; +} + +static string +StrToU8(string str) +{ + string res = "["; + foreach(ch; str) + { + res ~= "'" ~ ch ~ "',"; + } + res ~= "]"; + + return res; +} + +void +Highlight(FlatBuffer* fb) +{ + const const(u8[])[]['z'-95] keywords = [ + '_'-95: [ + mixin(StrToU8("__FILE__")), + mixin(StrToU8("__FILE_FULL_PATH__")), + mixin(StrToU8("__FUNCTION__")), + mixin(StrToU8("__LINE__")), + mixin(StrToU8("__MODULE__")), + mixin(StrToU8("__PRETTY_FUNCTION__")), + mixin(StrToU8("__gshared")), + mixin(StrToU8("__parameters")), + mixin(StrToU8("__rvalue")), + mixin(StrToU8("__traits")), + mixin(StrToU8("__vector")), + ], + 'a'-95: [ + mixin(StrToU8("abstract")), + mixin(StrToU8("alias")), + mixin(StrToU8("align")), + mixin(StrToU8("asm")), + mixin(StrToU8("assert")), + mixin(StrToU8("auto")), + ], + 'b'-95: [ + mixin(StrToU8("bool")), + mixin(StrToU8("break")), + mixin(StrToU8("byte")), + ], + 'c'-95: [ + mixin(StrToU8("case")), + mixin(StrToU8("cast")), + mixin(StrToU8("catch")), + mixin(StrToU8("char")), + mixin(StrToU8("class")), + mixin(StrToU8("const")), + mixin(StrToU8("continue")), + ], + 'd'-95: [ + mixin(StrToU8("dchar")), + mixin(StrToU8("debug")), + mixin(StrToU8("default")), + mixin(StrToU8("delegate")), + mixin(StrToU8("deprecated")), + mixin(StrToU8("do")), + mixin(StrToU8("double")), + ], + 'e'-95: [ + mixin(StrToU8("else")), + mixin(StrToU8("enum")), + mixin(StrToU8("export")), + mixin(StrToU8("extern")), + ], + 'f'-95: [ + mixin(StrToU8("false")), + mixin(StrToU8("final")), + mixin(StrToU8("finally")), + mixin(StrToU8("float")), + mixin(StrToU8("for")), + mixin(StrToU8("foreach")), + mixin(StrToU8("foreach_reverse")), + mixin(StrToU8("function")), + ], + 'g'-95: [ + mixin(StrToU8("goto")), + ], + 'i'-95: [ + mixin(StrToU8("if")), + mixin(StrToU8("immutable")), + mixin(StrToU8("import")), + mixin(StrToU8("in")), + mixin(StrToU8("inout")), + mixin(StrToU8("int")), + mixin(StrToU8("interface")), + mixin(StrToU8("invariant")), + mixin(StrToU8("is")), + ], + 'l'-95: [ + mixin(StrToU8("lazy")), + mixin(StrToU8("long")), + ], + 'm'-95: [ + mixin(StrToU8("mixin")), + mixin(StrToU8("module")), + ], + 'n'-95: [ + mixin(StrToU8("new")), + mixin(StrToU8("nothrow")), + mixin(StrToU8("null")), + ], + 'o'-95: [ + mixin(StrToU8("out")), + mixin(StrToU8("override")), + ], + 'p'-95: [ + mixin(StrToU8("package")), + mixin(StrToU8("pragma")), + mixin(StrToU8("private")), + mixin(StrToU8("protected")), + mixin(StrToU8("public")), + mixin(StrToU8("pure")), + ], + 'r'-95: [ + mixin(StrToU8("real")), + mixin(StrToU8("ref")), + mixin(StrToU8("return")), + ], + 's'-95: [ + mixin(StrToU8("scope")), + mixin(StrToU8("shared")), + mixin(StrToU8("short")), + mixin(StrToU8("static")), + mixin(StrToU8("struct")), + mixin(StrToU8("super")), + mixin(StrToU8("switch")), + mixin(StrToU8("synchronized")), + ], + 't'-95: [ + mixin(StrToU8("template")), + mixin(StrToU8("this")), + mixin(StrToU8("throw")), + mixin(StrToU8("true")), + mixin(StrToU8("try")), + mixin(StrToU8("typeid")), + mixin(StrToU8("typeof")), + ], + 'u'-95: [ + mixin(StrToU8("ubyte")), + mixin(StrToU8("uint")), + mixin(StrToU8("ulong")), + mixin(StrToU8("union")), + mixin(StrToU8("unittest")), + mixin(StrToU8("ushort")), + ], + 'v'-95: [ + mixin(StrToU8("version")), + mixin(StrToU8("void")), + ], + 'w'-95: [ + mixin(StrToU8("wchar")), + mixin(StrToU8("while")), + mixin(StrToU8("with")), + ], + ]; + + Tokenizer* tk = &fb.tk; + +TokenStream: + while (Advance(fb)) + { + u8[] token = fb.data[tk.pos .. tk.token_end]; + + if ((token[0] >= 'a' && token[0] <= 'z') || (token[0] >= 'A' && token[0] <= 'Z') || token[0] == '_') + { + u8 index = cast(u8)(token[0]-cast(u8)95); + if (index > 0 && index < keywords.length) + { + // TODO: look at simd string comparison + const(u8[])[] key_table = keywords[index]; + foreach(key; key_table) + { + if (token == key) + { + tk.buffer[tk.pos .. tk.token_end] = TT.Keyword; + continue TokenStream; + } + } + } + } + } +} + unittest { import std.stdio; diff --git a/src/editor/editor.d b/src/editor/editor.d index d3a636a..873b8eb 100644 --- a/src/editor/editor.d +++ b/src/editor/editor.d @@ -32,6 +32,7 @@ struct Editor u32 ui_count; FlatBuffer[] buffers; + Tokenizer[] tokenizers; u8[][] buffer_names; u32 buffer_count; FlatBuffer* active_buffer; @@ -65,11 +66,7 @@ Cycle(Editor* ed) f32 pos = 0.0; f32 h = ed.atlas_buf.atlas.size; - foreach(i, line; ed.linebufs.lines) - { - DrawText(ed, 0.0, pos, h, line[0 .. ed.linebufs.lengths[i]]); - pos += h; - } + DrawBuffer(ed, 0.0, 0.0, h, ed.active_buffer); BeginFrame(&ed.rd); @@ -159,10 +156,10 @@ CreateEditor(PlatformWindow* window, u8[] buffer_data, u8[] buffer_name) editor.buffer_count += 1; editor.active_buffer = &editor.buffers[0]; - - GetLines(editor.active_buffer, &editor.linebufs, 0, editor.active_buffer.line_count); } + editor.active_buffer = &editor.buffers[0]; + DescLayoutBinding[2] layout_bindings = [ { binding: 0, descriptorType: DT.Image, descriptorCount: 1, stageFlags: SS.All }, { binding: 1, descriptorType: DT.Storage, descriptorCount: 1, stageFlags: SS.All }, @@ -212,6 +209,54 @@ DrawText(Editor* ed, f32 x, f32 y, f32 px, string str) DrawText(ed, x, y, px, str_buf); } +void +DrawBuffer(Editor* ed, f32 x, f32 y, f32 px, FlatBuffer* fb) +{ + const Vec4[TT.max] cols = [ + TT.None: Vec4(1.0, 1.0, 1.0, 1.0), + TT.Keyword: Vec4(1.0, 0.0, 0.0, 1.0), + ]; + + u32 tab_count = 2; + f32 x_pos = x; + f32 y_pos = y; + f32 scale = px / ed.atlas_buf.atlas.size; + + foreach(i; 0 .. fb.length) + { + u8 ch = fb.data[i]; + + foreach(glyph; ed.atlas_buf.atlas.glyphs) + { + if (ch == glyph.ch) + { + if (ch == '\t') + { + Glyph g = glyph; + g.atlas_left = g.atlas_right = 0.0; + foreach(j; 0 .. tab_count) + { + DrawGlyph(ed, &g, scale, &x_pos, y_pos); + } + } + else if (ch == '\n') + { + Glyph g = glyph; + g.atlas_left = g.atlas_right = 0.0; + DrawGlyph(ed, &g, scale, &x_pos, y_pos); + y_pos += px; + x_pos = 0.0; + } + else + { + DrawGlyph(ed, &glyph, scale, &x_pos, y_pos, cols[fb.tk.buffer[i]]); + break; + } + } + } + } +} + void DrawText(Editor* ed, f32 x, f32 y, f32 px, u8[] str) { @@ -251,7 +296,7 @@ DrawText(Editor* ed, f32 x, f32 y, f32 px, u8[] str) } pragma(inline): void -DrawGlyph(Editor* ed, Glyph* glyph, f32 scale, f32* x_pos, f32 y) +DrawGlyph(Editor* ed, Glyph* glyph, f32 scale, f32* x_pos, f32 y, Vec4 col = Vec4(1.0)) { if (glyph.atlas_left != glyph.atlas_right) { @@ -273,11 +318,7 @@ DrawGlyph(Editor* ed, Glyph* glyph, f32 scale, f32* x_pos, f32 y) v.src_end.x = glyph.atlas_right; v.src_end.y = glyph.atlas_bottom; - v.col = Vec4(1.0, 1.0, 1.0, 1.0); - if (glyph.atlas_left == glyph.atlas_right) - { - v.col.r = v.col.g = v.col.b = 0.0; - } + v.col = col; AddUIIndices(ed); }