miki

commit 4cfb7f07f8cb88ca0a00bfd6d5a3c1e196a4b873
parent 06aaeebd50ef5350a6b45e367c8501c502c6b000
Author: weetat <wtenison@protonmail.com>
Date:   Tue, 25 Feb 2025 19:33:14 +0000

Finished first pass at 'fmd' lexer

Diffstat:
A
src/fmd/fmd.go
 | 
2
++
A
src/fmd/fmd_test.go
 | 
21
+++++++++++++++++++++
A
src/fmd/lexer.go
 | 
107
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
src/fmd/parser.go
 | 
5
+++++
M
src/miki/miki.go
 | 
3
++-

5 files changed, 137 insertions(+), 1 deletion(-)
diff --git a/src/fmd/fmd.go b/src/fmd/fmd.go
@@ -0,0 +1,2 @@
+package fmd
+
diff --git a/src/fmd/fmd_test.go b/src/fmd/fmd_test.go
@@ -0,0 +1,21 @@
+package fmd
+
+import (
+	"os"
+	"testing"
+)
+
+func TestLex(t *testing.T) {
+	_path := ("/home/weetat/MIKI/data/ents/home.fmd")
+	t.Log("Path:", _path)
+	_input, _err := os.ReadFile(_path)
+	if _err != nil {
+		t.Error("_ERROR\tcannot open file:", _err.Error())
+		return
+	}
+
+	_l := MakeLexer("MyLexer", _input)
+	_tokens := _l.Exec()
+
+	t.Log("Tokens:", _tokens)
+}
diff --git a/src/fmd/lexer.go b/src/fmd/lexer.go
@@ -0,0 +1,107 @@
+package fmd
+
+import (
+	"slices"
+	"strings"
+)
+
+/* Reference source: Lexical Scanning in Go - Rob Pike,
+https://www.youtube.com/watch?v=HxaD_trXwRE */
+
+type tokenType int
+
+const (
+	TokenLiteral tokenType = iota // (letters, numbers, symbols)+
+	TokenKeyword                  // (valid keywords)
+	TokenSpace                    // " "
+	TokenNewline                  // \n
+	TokenEscape                   // \
+	TokenColon                    // :
+	TokenLBrace                   // {
+	TokenRBrace                   // }
+)
+
+type token struct {
+	Type  tokenType
+	Value string
+}
+
+// var opSymbols = []string{":", "{", "}", "\\", "\\n"}
+var opKeywords = []string{"b", "i", "h1", "h2", "h3", "h4", "h5", "h6", "@", "#"}
+
+type lexer struct {
+	name   string // for error output
+	input  []byte // text input to scan
+	pos    int    // the start position for scanning
+	tokens []token
+}
+
+func MakeLexer(name string, input []byte) *lexer {
+	return &lexer{name: name, input: input, pos: 0}
+}
+
+func (l *lexer) Exec() []token {
+	for l.pos < len(l.input) {
+		_currentSymbol := l.input[l.pos]
+
+		switch {
+		case _currentSymbol == ' ':
+			l.tokens = append(l.tokens, token{TokenSpace, " "})
+			l.pos++
+		case _currentSymbol == '\n':
+			l.tokens = append(l.tokens, token{TokenNewline, "\n"})
+			l.pos++
+		case _currentSymbol == ':':
+			l.tokens = append(l.tokens, token{TokenColon, ":"})
+			l.pos++
+		case _currentSymbol == '{':
+			l.tokens = append(l.tokens, token{TokenLBrace, "{"})
+			l.pos++
+		case _currentSymbol == '}':
+			l.tokens = append(l.tokens, token{TokenRBrace, "}"})
+			l.pos++
+		case _currentSymbol == '\\':
+			l.tokens = append(l.tokens, token{TokenEscape, "\\"})
+			l.pos++
+		default:
+			{ // NOTE: inlined check until good reason to abstract arrives.
+				var _tokenString string
+				_start := l.pos
+
+				for l.pos < len(l.input) && !strings.Contains(":{} \\", string(l.input[l.pos])) {
+					l.pos++
+				}
+
+				_tokenString = string(l.input[_start:l.pos])
+
+				if slices.Contains(opKeywords, _tokenString) {
+					l.tokens = append(l.tokens, token{TokenKeyword, _tokenString})
+				} else if _tokenString != "" {
+					l.tokens = append(l.tokens, token{TokenLiteral, _tokenString})
+				}
+			}
+		}
+
+	}
+
+	return l.tokens
+}
+
+// func (l *lexer) checkWord() *token {
+// 	_start := l.pos
+
+// 	for l.pos < len(l.input) && !strings.Contains(":{} \\", string(l.input[l.pos])) {
+// 		l.pos++
+// 	}
+
+// 	_token := l.input[_start:l.pos]
+// 	if _token == "" {
+// 		return nil
+// 	}
+
+// 	if slices.Contains(opKeywords, _token) {
+// 		return &token{TokenKeyword, _token}
+// 	}
+
+// 	return &token{TokenLiteral, _token}
+// }
diff --git a/src/fmd/parser.go b/src/fmd/parser.go
@@ -0,0 +1,5 @@
+package fmd
+
+/* Reference source: Recursive descent parser,
+https://en.wikipedia.org/wiki/Recursive_descent_parser */
+
diff --git a/src/miki/miki.go b/src/miki/miki.go
@@ -10,4 +10,4 @@ func main() {
 
 	mnw.Init()
 	mnw.Exec()
-}
+}
+\ No newline at end of file