sleepy-day/editor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reimplement syntax highlighting

Browse Source
This commit is contained in:
Matthew 2025-09-28 19:37:28 +10:00
parent 6905c73710
commit 88e8f3325b
6 changed files with 755 additions and 787 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

443
src/editor/buffer.d
View File

@ -30,6 +30,7 @@ struct LineBuffers
struct LineBuffer
{
u8[] text;
TS[] style;
LineBuffer* next;
}
@ -45,7 +46,7 @@ CreateFlatBuffer(u8[] data)
{
u64 capacity = data.length > 0 ? RoundUp(cast(u64)(data.length) * 2, KB(4)) : KB(4);
u8[] buf = MAllocArray!(u8)(capacity);
u8[] buf = Alloc!(u8)(capacity);
buf[0 .. data.length] = data[0 .. data.length];
FlatBuffer fb = {
@ -66,16 +67,16 @@ CreateFlatBuffer(u8[] data)
void
Change(FlatBuffer* fb, u8[] data)
{
MFreeArray(fb.data);
FreeArray(fb.data);
u64 cap = data.length > 0 ? RoundUp(cast(u64)(data.length) * 2, KB(4)) : KB(4);
fb.data = MAllocArray!(u8)(cap);
fb.data = Alloc!(u8)(cap);
fb.data[0 .. data.length] = data[0 .. $];
fb.length = data.length;
fb.lf_count = CountLF(data);
Change(fb);
ResetTokenizer(fb);
Fix(fb);
}
@ -116,8 +117,7 @@ Fix(FlatBuffer* buffer)
dirty = false;
// TODO: fix
//Tokenize(buffer);
TokenizeD(buffer);
}
}
@ -259,6 +259,13 @@ GetLines(FlatBuffer* buffer, LineBuffers* linebufs, u64 start_line, u64 length)
linebufs.first.text = AllocArray!(u8)(&linebufs.arena, len);
linebufs.first.text = data[start_of_line .. start_of_line+len];
if(!Nil(buffer.tk.first))
{
linebufs.first.style = Alloc!(TS)(&linebufs.arena, len);
linebufs.first.style = buffer.tk.buffer[start_of_line .. start_of_line+len];
}
linebufs.count += 1;
}
else with(linebufs)
@ -271,8 +278,15 @@ GetLines(FlatBuffer* buffer, LineBuffers* linebufs, u64 start_line, u64 length)
if(len > 0)
{
current.text = AllocArray!(u8)(&arena, len);
current.text = Alloc!(u8)(&arena, len);
current.text[0 .. len] = buffer.data[start .. start+len];
if(!Nil(buffer.tk.first))
{
current.style = Alloc!(TS)(&arena, len);
current.style[0 .. len] = buffer.tk.buffer[start .. start+len];
}
count += 1;
current.next = Alloc!(LineBuffer)(&arena);
@ -473,421 +487,6 @@ Replace(FlatBuffer* buffer, u8[] insert, u64 length, u64 pos, u64 delete_length)
Insert(buffer, insert, insert.length, pos);
}
enum TokenStyle : u8
{
None = 0,
Keyword,
Import,
ImportTarget,
Type,
Identifier,
Op,
Bracket,
Function,
Macro,
Comment,
String,
Number,
Char,
Exclamation,
}
alias TS = TokenStyle;
enum TokenType : u32
{
None = 0,
Keyword,
Type,
Identifier,
String,
Char,
Function,
LiteralStr,
Macro,
Exclamation,
At,
Dollar,
Percent,
Caret,
Ampersand,
Asterisk,
LeftBracket,
RightBracket,
VertBar,
LeftBrace,
RightBrace,
LeftSquareBrace,
RightSquareBrace,
Tilde,
BackTick,
Semicolon,
Hash,
Colon,
Dot,
Minus,
Plus,
Equals,
Underscore,
BackSlash,
Slash,
LessThan,
GreaterThan,
Question,
Comma,
Comment,
Number,
SingleQuote,
DoubleQuote,
}
alias TT = TokenType;
struct Token
{
TokenType type;
u64 start;
u64 end;
}
struct Tokenizer
{
Arena arena;
TokenStyle[] buffer;
Token[] tokens;
u64 tk_count;
u64 pos;
}
Tokenizer
CreateTokenizer(FlatBuffer* fb)
{
Tokenizer tokenizer = {
arena: CreateArena(MB(8)),
buffer: MAllocArray!(TS)(fb.data.length),
};
return tokenizer;
}
void
Change(FlatBuffer* fb)
{
MFreeArray(fb.tk.buffer);
fb.tk.buffer = MAllocArray!(TS)(fb.data.length);
}
u8
PeekNextChar(FlatBuffer* fb)
{
return fb.tk.pos+1 < fb.length ? fb.data[fb.tk.pos+1] : 0;
}
/*
bool
Advance(FlatBuffer* fb)
{
Tokenizer* tk = &fb.tk;
tk.pos = tk.pos_end;
tk.next_type = TS.None;
u8[] buf = fb.data;
tk.pos_next = tk.pos;
tk.pos_end_next = tk.pos_end;
tk.pos = tk.pos == -1 ? 0 : tk.pos;
tk.pos_end = tk.pos_end == -1 ? 0 : tk.pos_end;
bool result = false;
bool started = false;
bool str_started = false;
for(i64 i = tk.pos; i < buf.length; i += 1)
{
u8 ch = buf.ptr[i];
bool space = CheckWhiteSpace(ch);
bool delim = CheckDelimiter(ch);
bool str = CheckString(ch);
if(!started && delim)
{
tk.pos = i;
tk.pos_end = i+1;
result = true;
break;
}
if(!started && !space)
{
tk.pos = i;
started = true;
str_started = str;
continue;
}
if(started && (delim || space) && !str_started)
{
result = true;
tk.pos_end = i;
if(ch == EOF)
{
Logf("final token");
tk.final_token = true;
}
switch(ch)
{
case '(': tk.next_type = TS.Bracket; break;
case '!': tk.next_type = TS.Exclamation; break;
default: break;
}
break;
}
if(started && str_started && str)
{
result = true;
tk.pos_end = i+1;
break;
}
}
return result;
}
static string
StrToU8(string str)
{
string res = "[";
foreach(ch; str)
{
res ~= "'" ~ ch ~ "',";
}
res ~= "]";
return res;
}
void
Highlight(FlatBuffer* fb)
{
const const(u8[])[]['z'-95] keywords = [
'_'-95: [
cast(u8[])r"__FILE__",
cast(u8[])r"__FILE_FULL_PATH__",
cast(u8[])r"__FUNCTION__",
cast(u8[])r"__LINE__",
cast(u8[])r"__MODULE__",
cast(u8[])r"__PRETTY_FUNCTION__",
cast(u8[])r"__gshared",
cast(u8[])r"__parameters",
cast(u8[])r"__rvalue",
cast(u8[])r"__traits",
cast(u8[])r"__vector",
],
'a'-95: [
cast(u8[])r"abstract",
cast(u8[])r"alias",
cast(u8[])r"align",
cast(u8[])r"asm",
cast(u8[])r"assert",
cast(u8[])r"auto",
],
'b'-95: [
cast(u8[])r"bool",
cast(u8[])r"break",
cast(u8[])r"byte",
],
'c'-95: [
cast(u8[])r"case",
cast(u8[])r"cast",
cast(u8[])r"catch",
cast(u8[])r"char",
cast(u8[])r"class",
cast(u8[])r"const",
cast(u8[])r"continue",
],
'd'-95: [
cast(u8[])r"dchar",
cast(u8[])r"debug",
cast(u8[])r"default",
cast(u8[])r"delegate",
cast(u8[])r"deprecated",
cast(u8[])r"do",
cast(u8[])r"double",
],
'e'-95: [
cast(u8[])r"else",
cast(u8[])r"enum",
cast(u8[])r"export",
cast(u8[])r"extern",
],
'f'-95: [
cast(u8[])r"false",
cast(u8[])r"final",
cast(u8[])r"finally",
cast(u8[])r"float",
cast(u8[])r"for",
cast(u8[])r"foreach",
cast(u8[])r"foreach_reverse",
cast(u8[])r"function",
],
'g'-95: [
cast(u8[])r"goto",
],
'i'-95: [
cast(u8[])r"if",
cast(u8[])r"immutable",
cast(u8[])r"import",
cast(u8[])r"in",
cast(u8[])r"inout",
cast(u8[])r"int",
cast(u8[])r"interface",
cast(u8[])r"invariant",
cast(u8[])r"is",
],
'l'-95: [
cast(u8[])r"lazy",
cast(u8[])r"long",
],
'm'-95: [
cast(u8[])r"mixin",
cast(u8[])r"module",
],
'n'-95: [
cast(u8[])r"new",
cast(u8[])r"nothrow",
cast(u8[])r"null",
],
'o'-95: [
cast(u8[])r"out",
cast(u8[])r"override",
],
'p'-95: [
cast(u8[])r"package",
cast(u8[])r"pragma",
cast(u8[])r"private",
cast(u8[])r"protected",
cast(u8[])r"public",
cast(u8[])r"pure",
],
'r'-95: [
cast(u8[])r"real",
cast(u8[])r"ref",
cast(u8[])r"return",
],
's'-95: [
cast(u8[])r"scope",
cast(u8[])r"shared",
cast(u8[])r"short",
cast(u8[])r"static",
cast(u8[])r"struct",
cast(u8[])r"super",
cast(u8[])r"switch",
cast(u8[])r"synchronized",
],
't'-95: [
cast(u8[])r"template",
cast(u8[])r"this",
cast(u8[])r"throw",
cast(u8[])r"true",
cast(u8[])r"try",
cast(u8[])r"typeid",
cast(u8[])r"typeof",
],
'u'-95: [
cast(u8[])r"ubyte",
cast(u8[])r"uint",
cast(u8[])r"ulong",
cast(u8[])r"union",
cast(u8[])r"unittest",
cast(u8[])r"ushort",
],
'v'-95: [
cast(u8[])r"version",
cast(u8[])r"void",
],
'w'-95: [
cast(u8[])r"wchar",
cast(u8[])r"while",
cast(u8[])r"with",
],
];
Tokenizer* tk = &fb.tk;
u8[] token;
u8[] prev_token;
if(tk.final_token)
{
goto InnerTokenStream;
}
TokenStream:
while (Advance(fb))
{
InnerTokenStream:
token = fb.data[tk.pos .. tk.pos_end];
scope(exit) prev_token = token;
if(token.length == 0) break;
if(token.length == 1 && CheckDelimiter(token.ptr[0]))
{
tk.buffer[tk.pos] = TS.Bracket;
continue TokenStream;
}
if(CheckString(token[0]) && CheckString(token[token.length-1]) && token[0] == token[token.length-1])
{
tk.buffer[tk.pos .. tk.pos_end] = TS.String;
continue TokenStream;
}
if(token[0] >= '0' && token[0] <= '9')
{
tk.buffer[tk.pos .. tk.pos_end] = TS.Number;
continue TokenStream;
}
if((token[0] >= 'a' && token[0] <= 'z') || (token[0] >= 'A' && token[0] <= 'Z') || token[0] == '_')
{
if(prev_token == r"import")
{
tk.buffer[tk.pos .. tk.pos_end] = TS.ImportTarget;
continue TokenStream;
}
if(tk.next_type == TS.Exclamation)
{
tk.buffer[tk.pos .. tk.pos_end] = TS.Macro;
continue TokenStream;
}
u8 index = cast(u8)(token[0]-cast(u8)95);
if(index > 0 && index < keywords.length)
{
// TODO: look at simd string comparison
const(u8[])[] key_table = keywords[index];
foreach(key; key_table)
{
if(token == key)
{
tk.buffer[tk.pos .. tk.pos_end] = TS.Keyword;
continue TokenStream;
}
}
}
tk.buffer[tk.pos .. tk.pos_end] = (tk.next_type == TS.Bracket) ? TS.Function : TS.Identifier;
continue TokenStream;
}
}
}
*/
unittest
{
import std.stdio;

21
src/editor/editor.d
View File

@ -1,15 +1,12 @@
import dlib.aliases;
import dlib.util;
import dlib.platform;
import dlib.fonts;
import dlib;
import vulkan;
import dlib.math;
import std.format : sformat;
import dlib.alloc;
import buffer;
import ui;
import widgets : Nil;
import widgets;
import parsing;
import std.format;
import std.stdio;
@ -155,7 +152,7 @@ InitEditorCtx(PlatformWindow* window)
cmd: {
arena: CreateArena(MB(1)),
cmd_arena: CreateArena(MB(1)),
buffer: MAllocArray!(u8)(1024),
buffer: Alloc!(u8)(1024),
},
};
@ -227,23 +224,19 @@ OpenFile(Editor* ed, u8[] file_name)
auto f = fopen(cast(char*)file_name.ptr, "rb");
if(f != null)
{
Logf("opened");
scope(exit) fclose(f);
fseek(f, 0, SEEK_END);
i64 len = ftell(f);
fseek(f, 0, SEEK_SET);
if(len > 0)
{
Logf("b");
u8[] buf = ScratchAlloc!(u8)(len);
Logf("a");
fread(buf.ptr, u8.sizeof, len, f);
Change(&ed.buf, buf);
Logf("read");
Change(&ed.buf, buf);
}
fclose(f);
}
else
{

2
src/editor/main.d
View File

@ -13,7 +13,7 @@ void main(string[] argv)
{
buffer_name = (cast(u8*)argv[1].ptr)[0 .. argv[1].length];
File f = File(argv[1], "rb");
buffer = MAllocArray!(u8)(f.size());
buffer = Alloc!(u8)(f.size());
f.rawRead(buffer);
f.close();
}

980
src/editor/parsing.d
View File

File diff suppressed because it is too large Load Diff

47
src/editor/ui.d
View File

@ -10,6 +10,7 @@ import std.format : sformat;
import core.stdc.string : memset;
import core.stdc.math : fabsf;
import parsing;
import editor;
const u8[] FONT_BYTES = import("pc-9800.ttf");
@ -179,6 +180,7 @@ struct UIItem
// Calculated Parameters
Vec4[4] color;
Vec4[4] border_color;
TokenStyle[] token_styles;
Rect rect;
Vec2 size;
Vec2 last_click_pos;
@ -1227,14 +1229,20 @@ GlyphWidth(Glyph* g)
return width;
}
Node!(u8[])*
MakeMultiline(u8[] text, f32 width, u8[] parent_id, u64 line_no)
struct TextBuffer
{
u8[] text;
TS[] style;
}
Node!(TextBuffer)*
MakeMultiline(u8[] text, f32 width, u8[] parent_id, u64 line_no, TS[] style = [])
{
f32 scaled_width = width * (g_ui_ctx.atlas_buf.atlas.size/g_ui_ctx.text_size);
f32 text_width = CalcTextWidth(text);
u64 line_count = cast(u64)(ceil(text_width/scaled_width));
Node!(u8[])* node = null;
Node!(TextBuffer)* node = null;
if(line_count > 0)
{
f32 w = 0.0;
@ -1252,25 +1260,39 @@ MakeMultiline(u8[] text, f32 width, u8[] parent_id, u64 line_no)
(cast(char[])buf).sformat("%s%05s%05s", cast(char[])parent_id, line_no, line);
u8[] str = ScratchAlloc!(u8)(len+extra_buf);
str[0 .. len] = text[start .. start+len];
str[len .. len+extra_buf] = buf[0 .. $];
Node!(u8[])* n = node;
TS[] stl = [];
if(style.length > 0)
{
stl = ScratchAlloc!(TS)(len);
stl[0 .. len] = style[start .. start+len];
}
Node!(TextBuffer)* n = node;
for(;;)
{
if(node == null)
{
node = ScratchAlloc!(Node!(u8[]))();
node.value = str;
node = ScratchAlloc!(Node!(TextBuffer))();
node.value.text = str;
node.value.style = stl;
node.next = null;
break;
}
if(n.next == null)
{
n.next = ScratchAlloc!(Node!(u8[]))();
n.next.value = str;
n.next = ScratchAlloc!(Node!(TextBuffer))();
n.next.value.text = str;
n.next.value.style = stl;
n.next.next = null;
break;
}
@ -1338,9 +1360,10 @@ DrawLine(UIItem* item)
for(u64 i = 0; i < item.key.text.length && item.key.text[i] != '\0'; i += 1)
{
u8 ch = item.key.text.ptr[i];
TS ts = i < item.token_styles.length ? item.token_styles[i] : TS.None;
if(ch < 128)
{
DrawGlyph(item, &atlas.glyphs[ch], atlas.size/ctx.text_size, &x, y, !edit_mode && i == item.highlighted_char);
DrawGlyph(item, &atlas.glyphs[ch], atlas.size/ctx.text_size, &x, y, !edit_mode && i == item.highlighted_char, ts);
}
}
@ -1430,7 +1453,7 @@ CullText(UIItem* item, GlyphBounds* gb)
}
pragma(inline) void
DrawGlyph(UIItem* item, Glyph* glyph, f32 scale, f32* x_pos, f32 y, bool highlight = false, Vec4 col = Vec4(1.0))
DrawGlyph(UIItem* item, Glyph* glyph, f32 scale, f32* x_pos, f32 y, bool highlight = false, TokenStyle ts = TS.None, Vec4 col = Vec4(1.0))
{
UICtx* ctx = GetCtx();
Vertex* bg_v = null, v = null;
@ -1484,6 +1507,10 @@ DrawGlyph(UIItem* item, Glyph* glyph, f32 scale, f32* x_pos, f32 y, bool highlig
{
col = Vec4(Vec3(1.0)-col.xyz, 1.0);
}
else if(ts != TS.None)
{
col = SYNTAX_COLORS[ts];
}
v.cols = col;

41
src/editor/widgets.d
View File

@ -3,6 +3,7 @@ import dlib;
import buffer;
import ui : Nil;
import ui;
import parsing;
import editor;
import std.format : sformat;
import std.math.rounding : ceil, floor;
@ -221,14 +222,19 @@ LineCounter(u8[] parent_id, FlatBuffer* buf, f32 offset, i64 start_row, i64 end_
{
UIItem* inner = Container(ScratchName(parent_id, "lcinner"), s_x, s_y, A2D.Y, UIF.None);
{
Push!("offset")(Vec2(0.0, offset));
for(u64 i = 0; i < line_counts.length; i += 1)
{
if(i == 0)
{
Push!("offset")(Vec2(0.0, offset));
UIItem* line = Text(line_counts[i]);
Pop!("offset");
}
else
{
UIItem* line = Text(line_counts[i]);
}
Pop!("offset")();
}
}
EndContainer();
}
@ -370,12 +376,15 @@ EditorView(UIPanel* panel)
Container(ScratchName(panel.id, "lines"), UISize(ST.Percentage, 1.0), UISize(ST.Percentage, 1.0), A2D.Y);
{
Push!("offset")(Vec2(0.0, offset));
U64Vec2 pos = VecPos(&ed.buf);
u64 i = 0;
for(LineBuffer* buf = ed.linebufs.first; buf != null; buf = buf.next, i += 1)
{
if(buf == ed.linebufs.first)
{
Push!("offset")(Vec2(0.0, offset));
}
if(buf.text.length > 0)
{
i64 line_no = i+line_offset;
@ -384,15 +393,19 @@ EditorView(UIPanel* panel)
Push!("highlighted_char")(pos.x);
}
TextPart* tp = WrappedTextLine(buf.text, panel.id, text_size, line_no);
TextPart* tp = WrappedTextLine(buf.text, panel.id, text_size, line_no, buf.style);
height += (text_size * tp.count);
if(TextClicked(tp))
{
SetFocusedPanel(panel);
}
}
Push!("highlighted_char")(-1);
if(buf == ed.linebufs.first)
{
Pop!("offset");
}
}
}
EndContainer();
@ -405,8 +418,6 @@ EditorView(UIPanel* panel)
{
SetFocusedPanel(panel);
}
Pop!("offset");
}
bool
@ -429,7 +440,7 @@ TextClicked(TextPart* text_part)
}
TextPart*
WrappedTextLine(u8[] text, u8[] parent_id, f32 text_size, u64 line_no)
WrappedTextLine(u8[] text, u8[] parent_id, f32 text_size, u64 line_no, TS[] style = [])
{
Push!("color")(Vec4(1.0));
Push!("text_size")(text_size);
@ -439,15 +450,17 @@ WrappedTextLine(u8[] text, u8[] parent_id, f32 text_size, u64 line_no)
part.item = g_UI_NIL;
TextPart* tp = part;
Node!(u8[])* lines = MakeMultiline(text, parent.size.x, parent_id, line_no);
for(Node!(u8[])* line = lines; line != null; line = line.next, tp = tp.next)
Node!(TextBuffer)* lines = MakeMultiline(text, parent.size.x, parent_id, line_no, style);
for(Node!(TextBuffer)* line = lines; line != null; line = line.next, tp = tp.next)
{
part.count += 1;
tp.item = Get(line.value);
tp.item = Get(line.value.text);
tp.next = ScratchAlloc!(TextPart)();
tp.next.item = g_UI_NIL;
tp.item.token_styles = line.value.style;
Signal(tp.item);
if(tp.item.signal & UIS.Clicked)