starting work on syntax highlighting

src/editor/buffer.d

@@ -2,9 +2,11 @@ import aliases;
 import math;
 import alloc;
 import util;
+import core.stdc.stdio : EOF;
 
 struct FlatBuffer
 {
+	Tokenizer tk;
 	Arena arena;
 	u8[]  data;
 	u64   length;
@@ -42,12 +44,18 @@ CreateFlatBuffer(u8[] data)
 		}
 	}
 
-	return FlatBuffer(
+	FlatBuffer fb = {
 		arena: CreateArena(MB(1)),
 		data: buf,
 		length: cast(u64)data.length,
 		line_count: line_count,
-	);
+	};
+
+	fb.tk = CreateTokenizer(&fb);
+
+	Highlight(&fb);
+
+	return fb;
 }
 
 LineBuffers
@@ -218,6 +226,261 @@ Replace(FlatBuffer* buffer, u8[] insert, u64 length, u64 pos, u64 delete_length)
 	Insert(buffer, insert, insert.length, pos);
 }
 
+enum TokenType : u8
+{
+	None = 0,
+	Keyword,
+	Import,
+	ImportTarget,
+	Type,
+	Identifier,
+	Op,
+	Bracket,
+	Function,
+	Macro,
+	Comment,
+	String,
+	Number,
+	Char,
+}
+
+alias TT = TokenType;
+
+struct Tokenizer
+{
+	TokenType[] buffer;
+	u64         pos;
+	u64         token_end;
+	u64         length;
+	TokenType   prev_token;
+	TokenType   current_token;
+}
+
+Tokenizer
+CreateTokenizer(FlatBuffer* buffer)
+{
+	Tokenizer tokenizer = {
+		buffer: MAllocArray!(TT)(buffer.data.length),
+	};
+
+	return tokenizer;
+}
+
+bool
+Advance(FlatBuffer* fb)
+{
+	Tokenizer* tk = &fb.tk;
+	tk.pos = tk.token_end;
+	u8[] buf = fb.data;
+
+	bool result = false;
+	bool started = false;
+	for(u64 i = tk.pos; i < buf.length; i += 1)
+	{
+		if (!started && buf.ptr[i] != ' ' && buf.ptr[i] != '\t' && buf.ptr[i] != '\n' && buf.ptr[i] != EOF)
+		{
+			tk.pos = i;
+			started = true;
+			continue;
+		}
+
+		if (started && (buf.ptr[i] == ' ' || buf.ptr[i] == '\t' || buf.ptr[i] == '\n' || buf.ptr[i] == EOF))
+		{
+			result = true;
+			tk.token_end = i;
+			break;
+		}
+	}
+
+	return result;
+}
+
+static string
+StrToU8(string str)
+{
+	string res = "[";
+	foreach(ch; str)
+	{
+		res ~= "'" ~ ch ~ "',";
+	}
+	res ~= "]";
+
+	return res;
+}
+
+void
+Highlight(FlatBuffer* fb)
+{
+	const const(u8[])[]['z'-95] keywords = [
+		'_'-95: [
+			mixin(StrToU8("__FILE__")),
+			mixin(StrToU8("__FILE_FULL_PATH__")),
+			mixin(StrToU8("__FUNCTION__")),
+			mixin(StrToU8("__LINE__")),
+			mixin(StrToU8("__MODULE__")),
+			mixin(StrToU8("__PRETTY_FUNCTION__")),
+			mixin(StrToU8("__gshared")),
+			mixin(StrToU8("__parameters")),
+			mixin(StrToU8("__rvalue")),
+			mixin(StrToU8("__traits")),
+			mixin(StrToU8("__vector")),
+		],
+		'a'-95: [
+			mixin(StrToU8("abstract")),
+			mixin(StrToU8("alias")),
+			mixin(StrToU8("align")),
+			mixin(StrToU8("asm")),
+			mixin(StrToU8("assert")),
+			mixin(StrToU8("auto")),
+		],
+		'b'-95: [
+			mixin(StrToU8("bool")),
+			mixin(StrToU8("break")),
+			mixin(StrToU8("byte")),
+		],
+		'c'-95: [
+			mixin(StrToU8("case")),
+			mixin(StrToU8("cast")),
+			mixin(StrToU8("catch")),
+			mixin(StrToU8("char")),
+			mixin(StrToU8("class")),
+			mixin(StrToU8("const")),
+			mixin(StrToU8("continue")),
+		],
+		'd'-95: [
+			mixin(StrToU8("dchar")),
+			mixin(StrToU8("debug")),
+			mixin(StrToU8("default")),
+			mixin(StrToU8("delegate")),
+			mixin(StrToU8("deprecated")),
+			mixin(StrToU8("do")),
+			mixin(StrToU8("double")),
+		],
+		'e'-95: [
+			mixin(StrToU8("else")),
+			mixin(StrToU8("enum")),
+			mixin(StrToU8("export")),
+			mixin(StrToU8("extern")),
+		],
+		'f'-95: [
+			mixin(StrToU8("false")),
+			mixin(StrToU8("final")),
+			mixin(StrToU8("finally")),
+			mixin(StrToU8("float")),
+			mixin(StrToU8("for")),
+			mixin(StrToU8("foreach")),
+			mixin(StrToU8("foreach_reverse")),
+			mixin(StrToU8("function")),
+		],
+		'g'-95: [
+			mixin(StrToU8("goto")),
+		],
+		'i'-95: [
+			mixin(StrToU8("if")),
+			mixin(StrToU8("immutable")),
+			mixin(StrToU8("import")),
+			mixin(StrToU8("in")),
+			mixin(StrToU8("inout")),
+			mixin(StrToU8("int")),
+			mixin(StrToU8("interface")),
+			mixin(StrToU8("invariant")),
+			mixin(StrToU8("is")),
+		],
+		'l'-95: [
+			mixin(StrToU8("lazy")),
+			mixin(StrToU8("long")),
+		],
+		'm'-95: [
+			mixin(StrToU8("mixin")),
+			mixin(StrToU8("module")),
+		],
+		'n'-95: [
+			mixin(StrToU8("new")),
+			mixin(StrToU8("nothrow")),
+			mixin(StrToU8("null")),
+		],
+		'o'-95: [
+			mixin(StrToU8("out")),
+			mixin(StrToU8("override")),
+		],
+		'p'-95: [
+			mixin(StrToU8("package")),
+			mixin(StrToU8("pragma")),
+			mixin(StrToU8("private")),
+			mixin(StrToU8("protected")),
+			mixin(StrToU8("public")),
+			mixin(StrToU8("pure")),
+		],
+		'r'-95: [
+			mixin(StrToU8("real")),
+			mixin(StrToU8("ref")),
+			mixin(StrToU8("return")),
+		],
+		's'-95: [
+			mixin(StrToU8("scope")),
+			mixin(StrToU8("shared")),
+			mixin(StrToU8("short")),
+			mixin(StrToU8("static")),
+			mixin(StrToU8("struct")),
+			mixin(StrToU8("super")),
+			mixin(StrToU8("switch")),
+			mixin(StrToU8("synchronized")),
+		],
+		't'-95: [
+			mixin(StrToU8("template")),
+			mixin(StrToU8("this")),
+			mixin(StrToU8("throw")),
+			mixin(StrToU8("true")),
+			mixin(StrToU8("try")),
+			mixin(StrToU8("typeid")),
+			mixin(StrToU8("typeof")),
+		],
+		'u'-95: [
+			mixin(StrToU8("ubyte")),
+			mixin(StrToU8("uint")),
+			mixin(StrToU8("ulong")),
+			mixin(StrToU8("union")),
+			mixin(StrToU8("unittest")),
+			mixin(StrToU8("ushort")),
+		],
+		'v'-95: [
+			mixin(StrToU8("version")),
+			mixin(StrToU8("void")),
+		],
+		'w'-95: [
+			mixin(StrToU8("wchar")),
+			mixin(StrToU8("while")),
+			mixin(StrToU8("with")),
+		],
+	];
+
+	Tokenizer* tk = &fb.tk;
+
+TokenStream:
+	while (Advance(fb))
+	{
+		u8[] token = fb.data[tk.pos .. tk.token_end];
+
+		if ((token[0] >= 'a' && token[0] <= 'z') || (token[0] >= 'A' && token[0] <= 'Z') || token[0] == '_')
+		{
+			u8 index = cast(u8)(token[0]-cast(u8)95);
+			if (index > 0 && index < keywords.length)
+			{
+				// TODO: look at simd string comparison
+				const(u8[])[] key_table = keywords[index];
+				foreach(key; key_table)
+				{
+					if (token == key)
+					{
+						tk.buffer[tk.pos .. tk.token_end] = TT.Keyword;
+						continue TokenStream;
+					}
+				}
+			}
+		}
+	}
+}
+
 unittest
 {
 	import std.stdio;

src/editor/editor.d

@@ -32,6 +32,7 @@ struct Editor
 	u32                   ui_count;
 
 	FlatBuffer[]        buffers;
+	Tokenizer[]         tokenizers;
 	u8[][]	        buffer_names;
 	u32             buffer_count;
 	FlatBuffer*     active_buffer;
@@ -65,11 +66,7 @@ Cycle(Editor* ed)
 
 	f32 pos = 0.0;
 	f32 h = ed.atlas_buf.atlas.size;
-	foreach(i, line; ed.linebufs.lines)
+	DrawBuffer(ed, 0.0, 0.0, h, ed.active_buffer);
-	{
-		DrawText(ed, 0.0, pos, h, line[0 .. ed.linebufs.lengths[i]]);
-		pos += h;
-	}
 
 	BeginFrame(&ed.rd);
 	
@@ -159,10 +156,10 @@ CreateEditor(PlatformWindow* window, u8[] buffer_data, u8[] buffer_name)
 
 		editor.buffer_count += 1;
 		editor.active_buffer = &editor.buffers[0];
-
-		GetLines(editor.active_buffer, &editor.linebufs, 0, editor.active_buffer.line_count);
 	}
 
+	editor.active_buffer = &editor.buffers[0];
+
 	DescLayoutBinding[2] layout_bindings = [
 		{ binding: 0, descriptorType: DT.Image, descriptorCount: 1, stageFlags: SS.All },
 		{ binding: 1, descriptorType: DT.Storage, descriptorCount: 1, stageFlags: SS.All },
@@ -212,6 +209,54 @@ DrawText(Editor* ed, f32 x, f32 y, f32 px, string str)
 	DrawText(ed, x, y, px, str_buf);
 }
 
+void
+DrawBuffer(Editor* ed, f32 x, f32 y, f32 px, FlatBuffer* fb)
+{
+	const Vec4[TT.max] cols = [
+		TT.None: Vec4(1.0, 1.0, 1.0, 1.0),
+		TT.Keyword: Vec4(1.0, 0.0, 0.0, 1.0),
+	];
+
+	u32 tab_count = 2;
+	f32 x_pos = x;
+	f32 y_pos = y;
+	f32 scale = px / ed.atlas_buf.atlas.size;
+
+	foreach(i; 0 .. fb.length)
+	{
+		u8 ch = fb.data[i];
+		
+		foreach(glyph; ed.atlas_buf.atlas.glyphs)
+		{
+			if (ch == glyph.ch)
+			{
+				if (ch == '\t')
+				{
+					Glyph g = glyph;
+					g.atlas_left = g.atlas_right = 0.0;
+					foreach(j; 0 .. tab_count)
+					{
+						DrawGlyph(ed, &g, scale, &x_pos, y_pos);
+					}
+				}
+				else if (ch == '\n')
+				{
+					Glyph g = glyph;
+					g.atlas_left = g.atlas_right = 0.0;
+					DrawGlyph(ed, &g, scale, &x_pos, y_pos);
+					y_pos += px;
+					x_pos = 0.0;
+				}
+				else
+				{
+					DrawGlyph(ed, &glyph, scale, &x_pos, y_pos, cols[fb.tk.buffer[i]]);
+					break;
+				}
+			}
+		}
+	}
+}
+
 void
 DrawText(Editor* ed, f32 x, f32 y, f32 px, u8[] str)
 {
@@ -251,7 +296,7 @@ DrawText(Editor* ed, f32 x, f32 y, f32 px, u8[] str)
 }
 
 pragma(inline): void
-DrawGlyph(Editor* ed, Glyph* glyph, f32 scale, f32* x_pos, f32 y)
+DrawGlyph(Editor* ed, Glyph* glyph, f32 scale, f32* x_pos, f32 y, Vec4 col = Vec4(1.0))
 {
 	if (glyph.atlas_left != glyph.atlas_right)
 	{
@@ -273,11 +318,7 @@ DrawGlyph(Editor* ed, Glyph* glyph, f32 scale, f32* x_pos, f32 y)
 		v.src_end.x = glyph.atlas_right;
 		v.src_end.y = glyph.atlas_bottom;
 
-		v.col = Vec4(1.0, 1.0, 1.0, 1.0);
+		v.col = col;
-		if (glyph.atlas_left == glyph.atlas_right)
-		{
-			v.col.r = v.col.g = v.col.b = 0.0;
-		}
 
 		AddUIIndices(ed);
 	}