miki

commit 0a45c4708fa283f94989fe6324f148cfd16ce173
parent 7de8b91547a216dffde3a17898b377d8002214c8
Author: weetat <wtenison@protonmail.com>
Date:   Tue, 25 Feb 2025 23:50:35 +0000

Finish first pass of the fmd parser

Implement literal parsing, skip whitespace tokens inside blocks and move
the command lookahead into isNextCommand(). The lexer now ends words at
newlines, and the test builds the AST as well.

Diffstat:
M
src/fmd/fmd_test.go
 | 
7
++++++-
M
src/fmd/lexer.go
 | 
2
+-
M
src/fmd/parser.go
 | 
64
++++++++++++++++++++++++++++++++++++++++++++++++----------------
M
src/miki/miki.go
 | 
3
+--

4 files changed, 56 insertions(+), 20 deletions(-)
diff --git a/src/fmd/fmd_test.go b/src/fmd/fmd_test.go
@@ -5,7 +5,7 @@ import (
 	"testing"
 )
 
-func TestLex(t *testing.T) {
+func TestFMD(t *testing.T) {
 	_path := ("/home/weetat/MIKI/data/ents/home.fmd")
 	t.Log("Path:", _path)
 	_input, _err := os.ReadFile(_path)
@@ -18,4 +18,9 @@ func TestLex(t *testing.T) {
 	_tokens := _l.Exec()
 
 	t.Log("Tokens:", _tokens)
+
+	_p := MakeParser(_tokens)
+	_ast := _p.Exec()
+
+	t.Log("AST:", _ast)
 }
diff --git a/src/fmd/lexer.go b/src/fmd/lexer.go
@@ -72,7 +72,7 @@ func (l *lexer) Exec() []token {
 				var _tokenString string
 				_start := l.pos
 
-				for l.pos < len(l.input) && !strings.Contains(":{} \\", string(l.input[l.pos])) {
+				for l.pos < len(l.input) && !strings.Contains(":{} \\\n", string(l.input[l.pos])) {
 					l.pos++
 				}
 
diff --git a/src/fmd/parser.go b/src/fmd/parser.go
@@ -2,6 +2,7 @@ package fmd
 
 import (
 	"fmt"
+	"log"
 )
 
 type nodeType string
@@ -10,6 +11,7 @@ const (
 	nodeFMD     nodeType = "fmd"
 	nodeBlock   nodeType = "block"
 	nodeCommand nodeType = "command"
+	nodeLiteral nodeType = "literal"
 	nodeSpace   nodeType = "space"
 )
 
@@ -31,12 +33,9 @@ func MakeParser(tokens []token) *parser {
 func (p *parser) Exec() node {
 	_fmd := node{nType: nodeFMD}
 	for p.pos < len(p.tokens) {
-		if p.peekNextN(TokenKeyword, 0) &&
-			p.peekNextN(TokenColon, 1) &&
-			p.peekNextN(TokenLiteral, 2) ||
-			p.peekNextN(TokenLBrace, 2) {
+		if p.isNextCommand() { // keyword:?
 			_fmd.children = append(_fmd.children, p.parseCommand())
-		} else {
+		} else { // block
 			_fmd.children = append(_fmd.children, p.parseBlock())
 		}
 	}
@@ -51,44 +50,77 @@ func (p *parser) parseCommand() node {
 
 	p.consume(TokenColon)
 
+	log.Printf("Command:%v", _command)
+	// keyword:{block}
 	if p.peekNextN(TokenLBrace, 0) {
-		_block := p.parseBlock()
-		_command.children = []node{_block}
+		p.consume(TokenLBrace)
+
+		// log.Printf("Command:branch:Block:%v", _command)
+		_command.children = []node{p.parseBlock()}
 
 		p.consume(TokenRBrace)
-	} else {
-		_literal := p.parseLiteral()
-		_command.children = []node{_literal}
+	} else { // keyword:"literal"
+		// log.Printf("Command:branch:Literal:%v", _command)
+		_command.children = []node{p.parseLiteral()}
 	}
 
+	log.Printf("Command:completed:%v", _command)
 	return _command
 }
 
 func (p *parser) parseBlock() node {
 	_block := node{nType: nodeBlock}
 
+	log.Printf("Block:%v", _block)
 	for p.pos < len(p.tokens) && p.current().Type != TokenRBrace {
-		if p.peekNextN(TokenKeyword, 0) && p.peekNextN(TokenColon, 1) {
+		if p.isNextCommand() { // keyword:?
+			// log.Printf("Block:branch:Command:%v", _block)
 			_block.children = append(_block.children, p.parseCommand())
-		} else if p.peekNextN(TokenSpace, 0) {
-			_block.children = append(_block.children, node{nType: nodeSpace, value: " "})
-		} else {
+		} else if p.peekNextN(TokenSpace, 0) { // " "
+			// log.Printf("Block:space%v", _block)
+			// _block.children = append(_block.children, node{nType: nodeSpace, value: " "})
+			p.pos++
+		} else if p.peekNextN(TokenNewline, 0) { // "\n"
+			// log.Printf("Block:newline:%v", _block)
+			// _block.children = append(_block.children, node{nType: nodeSpace, value: "\\n"})
+			p.pos++
+		} else if p.peekNextN(TokenTab, 0) { // "\t"
+			// log.Printf("Block:tab:%v", _block)
+			// _block.children = append(_block.children, node{nType: nodeSpace, value: "\\n"})
+			p.pos++
+		} else { // "literal"
+			// log.Printf("Block:branch:Literal:%v", _block)
 			_block.children = append(_block.children, p.parseLiteral())
 		}
 	}
 
+	log.Printf("Block:completed:%v", _block)
 	return _block
 }
 
 func (p *parser) parseLiteral() node {
-	return node{}
+	_literal := node{nType: nodeLiteral, value: p.consume(TokenLiteral).Value}
+	log.Printf("Literal:%v", _literal)
+	return _literal
+}
+
+func (p *parser) isNextCommand() bool {
+	return p.peekNextN(TokenKeyword, 0) &&
+		p.peekNextN(TokenColon, 1) &&
+		p.peekNextN(TokenLiteral, 2) ||
+		p.peekNextN(TokenLBrace, 2)
 }
 
 func (p *parser) peekNextN(t tokenType, n int) bool {
-	return p.pos+n < len(p.tokens) && p.tokens[p.pos+n].Type == t
+	_res := p.pos+n < len(p.tokens) && p.tokens[p.pos+n].Type == t
+	if p.pos+n < len(p.tokens) {
+		// log.Printf("\tPeek:inputToken:%v, parserToken:%v", t, p.tokens[p.pos+n])
+	}
+	return _res
 }
 
 func (p *parser) consume(t tokenType) token {
+	// log.Printf("\tconsume:%v", t)
 	if p.peekNextN(t, 0) {
 		token := p.tokens[p.pos]
 		p.pos++
diff --git a/src/miki/miki.go b/src/miki/miki.go
@@ -10,4 +10,4 @@ func main() {
 
 	mnw.Init()
 	mnw.Exec()
-}
-\ No newline at end of file
+}