diff --git a/internal/ast/ast.go b/internal/ast/ast.go index c7731db9..c46febc1 100644 --- a/internal/ast/ast.go +++ b/internal/ast/ast.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - . "github.com/benhoyt/goawk/lexer" + "github.com/benhoyt/goawk/lexer" ) // Program is a parsed AWK program. @@ -175,21 +175,21 @@ func (e *IncrExpr) precedence() int { func (e *BinaryExpr) precedence() int { switch e.Op { - case AND: + case lexer.AND: return precAnd - case OR: + case lexer.OR: return precOr - case CONCAT: + case lexer.CONCAT: return precConcat - case ADD, SUB: + case lexer.ADD, lexer.SUB: return precAdd - case MUL, DIV, MOD: + case lexer.MUL, lexer.DIV, lexer.MOD: return precMul - case EQUALS, LESS, LTE, GREATER, GTE, NOT_EQUALS: + case lexer.EQUALS, lexer.LESS, lexer.LTE, lexer.GREATER, lexer.GTE, lexer.NOT_EQUALS: return precCompare - case MATCH, NOT_MATCH: + case lexer.MATCH, lexer.NOT_MATCH: return precMatch - case POW: + case lexer.POW: return precPower default: return precPrimary @@ -225,7 +225,7 @@ func (e *NamedFieldExpr) String() string { // UnaryExpr is an expression like -1234. type UnaryExpr struct { - Op Token + Op lexer.Token Value Expr } @@ -236,13 +236,13 @@ func (e *UnaryExpr) String() string { // BinaryExpr is an expression like 1 + 2. type BinaryExpr struct { Left Expr - Op Token + Op lexer.Token Right Expr } func (e *BinaryExpr) String() string { var op string - if e.Op == CONCAT { + if e.Op == lexer.CONCAT { op = " " } else { op = " " + e.Op.String() + " " @@ -254,7 +254,7 @@ func (e *BinaryExpr) String() string { type InExpr struct { Index []Expr Array string - ArrayPos Position + ArrayPos lexer.Position } func (e *InExpr) String() string { @@ -318,7 +318,7 @@ func (e *RegExpr) String() string { // VarExpr is a variable reference (special var, global, or local). type VarExpr struct { Name string - Pos Position + Pos lexer.Position } func (e *VarExpr) String() string { @@ -328,7 +328,7 @@ func (e *VarExpr) String() string { // IndexExpr is an expression like a[k] (rvalue or lvalue). type IndexExpr struct { Array string - ArrayPos Position + ArrayPos lexer.Position Index []Expr } @@ -353,7 +353,7 @@ func (e *AssignExpr) String() string { // AugAssignExpr is an assignment expression like x += 5. type AugAssignExpr struct { Left Expr // can be one of: var, array[x], $n - Op Token + Op lexer.Token Right Expr } @@ -364,7 +364,7 @@ func (e *AugAssignExpr) String() string { // IncrExpr is an increment or decrement expression like x++ or --y. type IncrExpr struct { Expr Expr - Op Token + Op lexer.Token Pre bool } @@ -378,7 +378,7 @@ func (e *IncrExpr) String() string { // CallExpr is a builtin function call like length($1). type CallExpr struct { - Func Token + Func lexer.Token Args []Expr } @@ -395,7 +395,7 @@ func (e *CallExpr) String() string { type UserCallExpr struct { Name string Args []Expr - Pos Position + Pos lexer.Position } func (e *UserCallExpr) String() string { @@ -472,8 +472,8 @@ func formatRegex(r string) string { // Stmt is the abstract syntax tree for any AWK statement. type Stmt interface { Node - StartPos() Position // position of first character belonging to the node - EndPos() Position // position of first character immediately after the node + StartPos() lexer.Position // position of first character belonging to the node + EndPos() lexer.Position // position of first character immediately after the node stmt() String() string } @@ -496,54 +496,54 @@ func (s *DeleteStmt) stmt() {} func (s *ReturnStmt) stmt() {} func (s *BlockStmt) stmt() {} -func (s *PrintStmt) StartPos() Position { return s.Start } -func (s *PrintfStmt) StartPos() Position { return s.Start } -func (s *ExprStmt) StartPos() Position { return s.Start } -func (s *IfStmt) StartPos() Position { return s.Start } -func (s *ForStmt) StartPos() Position { return s.Start } -func (s *ForInStmt) StartPos() Position { return s.Start } -func (s *WhileStmt) StartPos() Position { return s.Start } -func (s *DoWhileStmt) StartPos() Position { return s.Start } -func (s *BreakStmt) StartPos() Position { return s.Start } -func (s *ContinueStmt) StartPos() Position { return s.Start } -func (s *NextStmt) StartPos() Position { return s.Start } -func (s *NextfileStmt) StartPos() Position { return s.Start } -func (s *ExitStmt) StartPos() Position { return s.Start } -func (s *DeleteStmt) StartPos() Position { return s.Start } -func (s *ReturnStmt) StartPos() Position { return s.Start } -func (s *BlockStmt) StartPos() Position { return s.Start } - -func (s *PrintStmt) EndPos() Position { return s.End } -func (s *PrintfStmt) EndPos() Position { return s.End } -func (s *ExprStmt) EndPos() Position { return s.End } -func (s *IfStmt) EndPos() Position { return s.End } -func (s *ForStmt) EndPos() Position { return s.End } -func (s *ForInStmt) EndPos() Position { return s.End } -func (s *WhileStmt) EndPos() Position { return s.End } -func (s *DoWhileStmt) EndPos() Position { return s.End } -func (s *BreakStmt) EndPos() Position { return s.End } -func (s *ContinueStmt) EndPos() Position { return s.End } -func (s *NextStmt) EndPos() Position { return s.End } -func (s *NextfileStmt) EndPos() Position { return s.End } -func (s *ExitStmt) EndPos() Position { return s.End } -func (s *DeleteStmt) EndPos() Position { return s.End } -func (s *ReturnStmt) EndPos() Position { return s.End } -func (s *BlockStmt) EndPos() Position { return s.End } +func (s *PrintStmt) StartPos() lexer.Position { return s.Start } +func (s *PrintfStmt) StartPos() lexer.Position { return s.Start } +func (s *ExprStmt) StartPos() lexer.Position { return s.Start } +func (s *IfStmt) StartPos() lexer.Position { return s.Start } +func (s *ForStmt) StartPos() lexer.Position { return s.Start } +func (s *ForInStmt) StartPos() lexer.Position { return s.Start } +func (s *WhileStmt) StartPos() lexer.Position { return s.Start } +func (s *DoWhileStmt) StartPos() lexer.Position { return s.Start } +func (s *BreakStmt) StartPos() lexer.Position { return s.Start } +func (s *ContinueStmt) StartPos() lexer.Position { return s.Start } +func (s *NextStmt) StartPos() lexer.Position { return s.Start } +func (s *NextfileStmt) StartPos() lexer.Position { return s.Start } +func (s *ExitStmt) StartPos() lexer.Position { return s.Start } +func (s *DeleteStmt) StartPos() lexer.Position { return s.Start } +func (s *ReturnStmt) StartPos() lexer.Position { return s.Start } +func (s *BlockStmt) StartPos() lexer.Position { return s.Start } + +func (s *PrintStmt) EndPos() lexer.Position { return s.End } +func (s *PrintfStmt) EndPos() lexer.Position { return s.End } +func (s *ExprStmt) EndPos() lexer.Position { return s.End } +func (s *IfStmt) EndPos() lexer.Position { return s.End } +func (s *ForStmt) EndPos() lexer.Position { return s.End } +func (s *ForInStmt) EndPos() lexer.Position { return s.End } +func (s *WhileStmt) EndPos() lexer.Position { return s.End } +func (s *DoWhileStmt) EndPos() lexer.Position { return s.End } +func (s *BreakStmt) EndPos() lexer.Position { return s.End } +func (s *ContinueStmt) EndPos() lexer.Position { return s.End } +func (s *NextStmt) EndPos() lexer.Position { return s.End } +func (s *NextfileStmt) EndPos() lexer.Position { return s.End } +func (s *ExitStmt) EndPos() lexer.Position { return s.End } +func (s *DeleteStmt) EndPos() lexer.Position { return s.End } +func (s *ReturnStmt) EndPos() lexer.Position { return s.End } +func (s *BlockStmt) EndPos() lexer.Position { return s.End } // PrintStmt is a statement like print $1, $3. type PrintStmt struct { Args []Expr - Redirect Token + Redirect lexer.Token Dest Expr - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *PrintStmt) String() string { return printString("print", s.Args, s.Redirect, s.Dest) } -func printString(f string, args []Expr, redirect Token, dest Expr) string { +func printString(f string, args []Expr, redirect lexer.Token, dest Expr) string { parts := make([]string, len(args)) for i, a := range args { parts[i] = a.String() @@ -558,10 +558,10 @@ func printString(f string, args []Expr, redirect Token, dest Expr) string { // PrintfStmt is a statement like printf "%3d", 1234. type PrintfStmt struct { Args []Expr - Redirect Token + Redirect lexer.Token Dest Expr - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *PrintfStmt) String() string { @@ -571,8 +571,8 @@ func (s *PrintfStmt) String() string { // ExprStmt is statement like a bare function call: my_func(x). type ExprStmt struct { Expr Expr - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *ExprStmt) String() string { @@ -582,11 +582,11 @@ func (s *ExprStmt) String() string { // IfStmt is an if or if-else statement. type IfStmt struct { Cond Expr - BodyStart Position + BodyStart lexer.Position Body Stmts Else Stmts - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *IfStmt) String() string { @@ -602,10 +602,10 @@ type ForStmt struct { Pre Stmt Cond Expr Post Stmt - BodyStart Position + BodyStart lexer.Position Body Stmts - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *ForStmt) String() string { @@ -627,13 +627,13 @@ func (s *ForStmt) String() string { // ForInStmt is a for loop like for (k in a) print k, a[k]. type ForInStmt struct { Var string - VarPos Position + VarPos lexer.Position Array string - ArrayPos Position - BodyStart Position + ArrayPos lexer.Position + BodyStart lexer.Position Body Stmts - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *ForInStmt) String() string { @@ -643,10 +643,10 @@ func (s *ForInStmt) String() string { // WhileStmt is a while loop. type WhileStmt struct { Cond Expr - BodyStart Position + BodyStart lexer.Position Body Stmts - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *WhileStmt) String() string { @@ -657,8 +657,8 @@ func (s *WhileStmt) String() string { type DoWhileStmt struct { Body Stmts Cond Expr - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *DoWhileStmt) String() string { @@ -667,8 +667,8 @@ func (s *DoWhileStmt) String() string { // BreakStmt is a break statement. type BreakStmt struct { - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *BreakStmt) String() string { @@ -677,8 +677,8 @@ func (s *BreakStmt) String() string { // ContinueStmt is a continue statement. type ContinueStmt struct { - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *ContinueStmt) String() string { @@ -687,8 +687,8 @@ func (s *ContinueStmt) String() string { // NextStmt is a next statement. type NextStmt struct { - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *NextStmt) String() string { @@ -697,8 +697,8 @@ func (s *NextStmt) String() string { // NextfileStmt is a nextfile statement. type NextfileStmt struct { - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *NextfileStmt) String() string { @@ -708,8 +708,8 @@ func (s *NextfileStmt) String() string { // ExitStmt is an exit statement. type ExitStmt struct { Status Expr - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *ExitStmt) String() string { @@ -723,10 +723,10 @@ func (s *ExitStmt) String() string { // DeleteStmt is a statement like delete a[k]. type DeleteStmt struct { Array string - ArrayPos Position + ArrayPos lexer.Position Index []Expr - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *DeleteStmt) String() string { @@ -743,8 +743,8 @@ func (s *DeleteStmt) String() string { // ReturnStmt is a return statement. type ReturnStmt struct { Value Expr - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *ReturnStmt) String() string { @@ -758,8 +758,8 @@ func (s *ReturnStmt) String() string { // BlockStmt is a stand-alone block like { print "x" }. type BlockStmt struct { Body Stmts - Start Position - End Position + Start lexer.Position + End lexer.Position } func (s *BlockStmt) String() string { @@ -771,7 +771,7 @@ type Function struct { Name string Params []string Body Stmts - Pos Position + Pos lexer.Position } func (f *Function) String() string { @@ -782,13 +782,13 @@ func (f *Function) String() string { // PositionError represents an error bound to specific position in source. type PositionError struct { // Source line/column position where the error occurred. - Position Position + Position lexer.Position // Error message. Message string } // PosErrorf like fmt.Errorf, but with an explicit position. -func PosErrorf(pos Position, format string, args ...any) error { +func PosErrorf(pos lexer.Position, format string, args ...any) error { message := fmt.Sprintf(format, args...) return &PositionError{pos, message} } diff --git a/internal/parseutil/filereader_test.go b/internal/parseutil/filereader_test.go index 19f5f4d5..c2622ff4 100644 --- a/internal/parseutil/filereader_test.go +++ b/internal/parseutil/filereader_test.go @@ -1,9 +1,10 @@ package parseutil_test import ( - . "github.com/benhoyt/goawk/internal/parseutil" "strings" "testing" + + "github.com/benhoyt/goawk/internal/parseutil" ) type testFile struct{ name, source string } @@ -110,7 +111,7 @@ print x for _, tst := range tests { t.Run(tst.name, func(t *testing.T) { - fr := &FileReader{} + fr := &parseutil.FileReader{} for _, file := range tst.files { if nil != fr.AddFile(file.name, strings.NewReader(file.source)) { diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 42b3865f..db3dc353 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - . "github.com/benhoyt/goawk/lexer" + "github.com/benhoyt/goawk/lexer" ) func TestLexer(t *testing.T) { @@ -76,14 +76,14 @@ func TestLexer(t *testing.T) { } for _, test := range tests { t.Run(test.input, func(t *testing.T) { - l := NewLexer([]byte(test.input)) + l := lexer.NewLexer([]byte(test.input)) strs := []string{} for { pos, tok, val := l.Scan() - if tok == EOF { + if tok == lexer.EOF { break } - if tok == NUMBER { + if tok == lexer.NUMBER { // Ensure ParseFloat() works, as that's what our // parser uses to convert trimmed := strings.TrimRight(val, "eE") @@ -93,7 +93,7 @@ func TestLexer(t *testing.T) { } } strs = append(strs, fmt.Sprintf("%d:%d %s %q", pos.Line, pos.Column, tok, val)) - if tok == ILLEGAL { + if tok == lexer.ILLEGAL { break } } @@ -119,7 +119,7 @@ func TestRegex(t *testing.T) { } for _, test := range tests { t.Run(test.input, func(t *testing.T) { - l := NewLexer([]byte(test.input)) + l := lexer.NewLexer([]byte(test.input)) l.Scan() // Scan first token (probably DIV) pos, tok, val := l.ScanRegex() output := fmt.Sprintf("%d:%d %s %q", pos.Line, pos.Column, tok, val) @@ -142,7 +142,7 @@ func TestScanRegexInvalid(t *testing.T) { t.Fatalf("expected panic of string type") } }() - l := NewLexer([]byte("foo/")) + l := lexer.NewLexer([]byte("foo/")) l.Scan() // Scan first token (NAME foo) l.ScanRegex() } @@ -150,19 +150,19 @@ func TestScanRegexInvalid(t *testing.T) { func TestHadSpace(t *testing.T) { tests := []struct { input string - tokens []Token + tokens []lexer.Token spaces []bool }{ - {`foo(x)`, []Token{NAME, LPAREN, NAME, RPAREN}, []bool{false, false, false, false}}, - {`foo (x) `, []Token{NAME, LPAREN, NAME, RPAREN}, []bool{false, true, false, false}}, - {` foo ( x ) `, []Token{NAME, LPAREN, NAME, RPAREN}, []bool{true, true, true, true}}, + {`foo(x)`, []lexer.Token{lexer.NAME, lexer.LPAREN, lexer.NAME, lexer.RPAREN}, []bool{false, false, false, false}}, + {`foo (x) `, []lexer.Token{lexer.NAME, lexer.LPAREN, lexer.NAME, lexer.RPAREN}, []bool{false, true, false, false}}, + {` foo ( x ) `, []lexer.Token{lexer.NAME, lexer.LPAREN, lexer.NAME, lexer.RPAREN}, []bool{true, true, true, true}}, } for _, test := range tests { t.Run(test.input, func(t *testing.T) { - l := NewLexer([]byte(test.input)) + l := lexer.NewLexer([]byte(test.input)) for i := 0; ; i++ { _, tok, _ := l.Scan() - if tok == EOF { + if tok == lexer.EOF { break } if tok != test.tokens[i] { @@ -177,13 +177,13 @@ func TestHadSpace(t *testing.T) { } func TestPeekByte(t *testing.T) { - l := NewLexer([]byte("foo()")) + l := lexer.NewLexer([]byte("foo()")) b := l.PeekByte() if b != 'f' { t.Errorf("expected 'f', got %q", b) } _, tok, _ := l.Scan() - if tok != NAME { + if tok != lexer.NAME { t.Errorf("expected name, got %s", tok) } b = l.PeekByte() @@ -191,11 +191,11 @@ func TestPeekByte(t *testing.T) { t.Errorf("expected '(', got %q", b) } _, tok, _ = l.Scan() - if tok != LPAREN { + if tok != lexer.LPAREN { t.Errorf("expected (, got %s", tok) } _, tok, _ = l.Scan() - if tok != RPAREN { + if tok != lexer.RPAREN { t.Errorf("expected ), got %s", tok) } b = l.PeekByte() @@ -207,17 +207,17 @@ func TestPeekByte(t *testing.T) { func TestKeywordToken(t *testing.T) { tests := []struct { name string - tok Token + tok lexer.Token }{ - {"print", PRINT}, - {"split", F_SPLIT}, - {"BEGIN", BEGIN}, - {"foo", ILLEGAL}, - {"GoAWK", ILLEGAL}, + {"print", lexer.PRINT}, + {"split", lexer.F_SPLIT}, + {"BEGIN", lexer.BEGIN}, + {"foo", lexer.ILLEGAL}, + {"GoAWK", lexer.ILLEGAL}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - tok := KeywordToken(test.name) + tok := lexer.KeywordToken(test.name) if tok != test.tok { t.Errorf("expected %v, got %v", test.tok, tok) } @@ -236,14 +236,14 @@ func TestAllTokens(t *testing.T) { "x \"str\\n\" 1234\n" + "` ." - strs := make([]string, 0, LAST+1) - seen := make([]bool, LAST+1) - l := NewLexer([]byte(input)) + strs := make([]string, 0, lexer.LAST+1) + seen := make([]bool, lexer.LAST+1) + l := lexer.NewLexer([]byte(input)) for { _, tok, _ := l.Scan() strs = append(strs, tok.String()) seen[int(tok)] = true - if tok == EOF { + if tok == lexer.EOF { break } } @@ -263,22 +263,22 @@ func TestAllTokens(t *testing.T) { } for i, s := range seen { - if !s && Token(i) != CONCAT && Token(i) != REGEX { - t.Errorf("token %s (%d) not seen", Token(i), i) + if !s && lexer.Token(i) != lexer.CONCAT && lexer.Token(i) != lexer.REGEX { + t.Errorf("token %s (%d) not seen", lexer.Token(i), i) } } - l = NewLexer([]byte(`/foo/`)) + l = lexer.NewLexer([]byte(`/foo/`)) _, tok1, _ := l.Scan() _, tok2, val := l.ScanRegex() - if tok1 != DIV || tok2 != REGEX || val != "foo" { + if tok1 != lexer.DIV || tok2 != lexer.REGEX || val != "foo" { t.Errorf(`expected / regex "foo", got %s %s %q`, tok1, tok2, val) } - l = NewLexer([]byte(`/=foo/`)) + l = lexer.NewLexer([]byte(`/=foo/`)) _, tok1, _ = l.Scan() _, tok2, val = l.ScanRegex() - if tok1 != DIV_ASSIGN || tok2 != REGEX || val != "=foo" { + if tok1 != lexer.DIV_ASSIGN || tok2 != lexer.REGEX || val != "=foo" { t.Errorf(`expected /= regex "=foo", got %s %s %q`, tok1, tok2, val) } } @@ -300,7 +300,7 @@ func TestUnescape(t *testing.T) { } for _, test := range tests { t.Run(test.input, func(t *testing.T) { - got, err := Unescape(test.input) + got, err := lexer.Unescape(test.input) if err != nil { if err.Error() != test.error { t.Fatalf("expected error %q, got %q", test.error, err) @@ -318,7 +318,7 @@ func TestUnescape(t *testing.T) { } func TestPositionString(t *testing.T) { - pos := Position{Line: 42, Column: 7} + pos := lexer.Position{Line: 42, Column: 7} expected := "42:7" if pos.String() != expected { t.Errorf("expected %q, got %q", expected, pos.String()) @@ -329,10 +329,10 @@ func benchmarkLexer(b *testing.B, repeat int, source string) { fullSource := []byte(strings.Repeat(source+"\n", repeat)) b.ResetTimer() for i := 0; i < b.N; i++ { - l := NewLexer(fullSource) + l := lexer.NewLexer(fullSource) for { _, tok, _ := l.Scan() - if tok == EOF || tok == ILLEGAL { + if tok == lexer.EOF || tok == lexer.ILLEGAL { break } } @@ -372,17 +372,17 @@ func BenchmarkRegex(b *testing.B) { fullSource := []byte(strings.Repeat(source+" ", 5)) b.ResetTimer() for i := 0; i < b.N; i++ { - l := NewLexer(fullSource) + l := lexer.NewLexer(fullSource) for { _, tok, _ := l.Scan() - if tok == EOF { + if tok == lexer.EOF { break } - if tok != DIV && tok != DIV_ASSIGN { + if tok != lexer.DIV && tok != lexer.DIV_ASSIGN { b.Fatalf("expected / or /=, got %s", tok) } _, tok, _ = l.ScanRegex() - if tok != REGEX { + if tok != lexer.REGEX { b.Fatalf("expected regex, got %s", tok) } } @@ -390,10 +390,10 @@ func BenchmarkRegex(b *testing.B) { } func Example() { - lexer := NewLexer([]byte(`$0 { print $1 }`)) + lex := lexer.NewLexer([]byte(`$0 { print $1 }`)) for { - pos, tok, val := lexer.Scan() - if tok == EOF { + pos, tok, val := lex.Scan() + if tok == lexer.EOF { break } fmt.Printf("%s %s %q\n", pos, tok, val) diff --git a/parser/parser.go b/parser/parser.go index 8aaf703a..7b6ed457 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -14,14 +14,14 @@ import ( "github.com/benhoyt/goawk/internal/ast" "github.com/benhoyt/goawk/internal/compiler" "github.com/benhoyt/goawk/internal/resolver" - . "github.com/benhoyt/goawk/lexer" + "github.com/benhoyt/goawk/lexer" ) // ParseError (actually *ParseError) is the type of error returned by // ParseProgram. type ParseError struct { // Source line/column position where the error occurred. - Position Position + Position lexer.Position // Error message. Message string } @@ -74,9 +74,9 @@ func ParseProgram(src []byte, config *ParserConfig) (prog *Program, err error) { } } }() - lexer := NewLexer(src) - p := parser{lexer: lexer} - p.multiExprs = make(map[*ast.MultiExpr]Position, 3) + lex := lexer.NewLexer(src) + p := parser{lexer: lex} + p.multiExprs = make(map[*ast.MultiExpr]lexer.Position, 3) p.next() // initialize p.tok @@ -117,11 +117,11 @@ func (p *Program) Disassemble(writer io.Writer) error { // Parser state type parser struct { // Lexer instance and current token values - lexer *Lexer - pos Position // position of last token (tok) - tok Token // last lexed token - prevTok Token // previously lexed token - val string // string value of last token (or "") + lexer *lexer.Lexer + pos lexer.Position // position of last token (tok) + tok lexer.Token // last lexed token + prevTok lexer.Token // previously lexed token + val string // string value of last token (or "") // Parsing state inAction bool // true if parsing an action (false in BEGIN or END) @@ -130,7 +130,7 @@ type parser struct { pendingGetlineLeft ast.Expr // saved expression to the left of | // Variable tracking and resolving - multiExprs map[*ast.MultiExpr]Position // tracks comma-separated expressions + multiExprs map[*ast.MultiExpr]lexer.Position // tracks comma-separated expressions } // Parse an entire AWK program. @@ -148,9 +148,9 @@ func (p *parser) program() *ast.Program { // major AWK implementations. needsTerminator := false - for p.tok != EOF { + for p.tok != lexer.EOF { if needsTerminator { - if !p.matches(NEWLINE, SEMICOLON) { + if !p.matches(lexer.NEWLINE, lexer.SEMICOLON) { panic(p.errorf("expected ; or newline between items")) } p.next() @@ -158,31 +158,31 @@ func (p *parser) program() *ast.Program { } p.optionalNewlines() switch p.tok { - case EOF: + case lexer.EOF: // End of file - case BEGIN: + case lexer.BEGIN: p.next() prog.Begin = append(prog.Begin, p.stmtsBrace()) - case END: + case lexer.END: p.next() prog.End = append(prog.End, p.stmtsBrace()) - case FUNCTION: + case lexer.FUNCTION: function := p.function() prog.Functions = append(prog.Functions, function) default: p.inAction = true // Allow empty pattern, normal pattern, or range pattern pattern := []ast.Expr{} - if !p.matches(LBRACE, EOF) { + if !p.matches(lexer.LBRACE, lexer.EOF) { pattern = append(pattern, p.expr()) } - if !p.matches(LBRACE, EOF, NEWLINE, SEMICOLON) { + if !p.matches(lexer.LBRACE, lexer.EOF, lexer.NEWLINE, lexer.SEMICOLON) { p.commaNewlines() pattern = append(pattern, p.expr()) } // Or an empty action (equivalent to { print $0 }) action := &ast.Action{Pattern: pattern} - if p.tok == LBRACE { + if p.tok == lexer.LBRACE { action.Stmts = p.stmtsBrace() } else { needsTerminator = true @@ -200,12 +200,12 @@ func (p *parser) program() *ast.Program { // Parse a list of statements. func (p *parser) stmts() ast.Stmts { switch p.tok { - case SEMICOLON: + case lexer.SEMICOLON: // This is so things like this parse correctly: // BEGIN { for (i=0; i<10; i++); print "x" } p.next() return nil - case LBRACE: + case lexer.LBRACE: return p.stmtsBrace() default: return []ast.Stmt{p.stmt()} @@ -214,18 +214,18 @@ func (p *parser) stmts() ast.Stmts { // Parse a list of statements surrounded in {...} braces. func (p *parser) stmtsBrace() ast.Stmts { - p.expect(LBRACE) + p.expect(lexer.LBRACE) p.optionalNewlines() ss := []ast.Stmt{} - for p.tok != RBRACE && p.tok != EOF { - if p.matches(SEMICOLON, NEWLINE) { + for p.tok != lexer.RBRACE && p.tok != lexer.EOF { + if p.matches(lexer.SEMICOLON, lexer.NEWLINE) { p.next() continue } ss = append(ss, p.stmt()) } - p.expect(RBRACE) - if p.tok == SEMICOLON { + p.expect(lexer.RBRACE) + if p.tok == lexer.SEMICOLON { p.next() } return ss @@ -235,7 +235,7 @@ func (p *parser) stmtsBrace() ast.Stmts { func (p *parser) simpleStmt() ast.Stmt { startPos := p.pos switch p.tok { - case PRINT, PRINTF: + case lexer.PRINT, lexer.PRINTF: op := p.tok p.next() args := p.exprList(p.printExpr) @@ -246,14 +246,14 @@ func (p *parser) simpleStmt() ast.Stmt { p.useMultiExpr(m) } } - redirect := ILLEGAL + redirect := lexer.ILLEGAL var dest ast.Expr - if p.matches(GREATER, APPEND, PIPE) { + if p.matches(lexer.GREATER, lexer.APPEND, lexer.PIPE) { redirect = p.tok p.next() dest = p.expr() } - if op == PRINT { + if op == lexer.PRINT { return &ast.PrintStmt{Args: args, Redirect: redirect, Dest: dest, Start: startPos, End: p.pos} } else { if len(args) == 0 { @@ -261,20 +261,20 @@ func (p *parser) simpleStmt() ast.Stmt { } return &ast.PrintfStmt{Args: args, Redirect: redirect, Dest: dest, Start: startPos, End: p.pos} } - case DELETE: + case lexer.DELETE: p.next() name, namePos := p.expectName() var index []ast.Expr - if p.tok == LBRACKET { + if p.tok == lexer.LBRACKET { p.next() index = p.exprList(p.expr) if len(index) == 0 { panic(p.errorf("expected expression instead of ]")) } - p.expect(RBRACKET) + p.expect(lexer.RBRACKET) } return &ast.DeleteStmt{Array: name, ArrayPos: namePos, Index: index, Start: startPos, End: p.pos} - case IF, FOR, WHILE, DO, BREAK, CONTINUE, NEXT, NEXTFILE, EXIT, RETURN: + case lexer.IF, lexer.FOR, lexer.WHILE, lexer.DO, lexer.BREAK, lexer.CONTINUE, lexer.NEXT, lexer.NEXTFILE, lexer.EXIT, lexer.RETURN: panic(p.errorf("expected print/printf, delete, or expression")) default: return &ast.ExprStmt{Expr: p.expr(), Start: startPos, End: p.pos} @@ -286,23 +286,23 @@ func (p *parser) stmt() ast.Stmt { var s ast.Stmt startPos := p.pos switch p.tok { - case IF: + case lexer.IF: p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) cond := p.expr() - p.expect(RPAREN) + p.expect(lexer.RPAREN) p.optionalNewlines() bodyStart := p.pos body := p.stmts() p.optionalNewlines() var elseBody ast.Stmts - if p.tok == ELSE { + if p.tok == lexer.ELSE { p.next() p.optionalNewlines() elseBody = p.stmts() } s = &ast.IfStmt{Cond: cond, BodyStart: bodyStart, Body: body, Else: elseBody, Start: startPos, End: p.pos} - case FOR: + case lexer.FOR: // Parse for statement, either "for in" or C-like for loop. // // FOR LPAREN NAME IN NAME RPAREN NEWLINE* stmts | @@ -311,12 +311,12 @@ func (p *parser) stmt() ast.Stmt { // [simpleStmt] RPAREN NEWLINE* stmts // p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) var pre ast.Stmt - if p.tok != SEMICOLON { + if p.tok != lexer.SEMICOLON { pre = p.simpleStmt() } - if pre != nil && p.tok == RPAREN { + if pre != nil && p.tok == lexer.RPAREN { // Match: for (var in array) body p.next() p.optionalNewlines() @@ -349,85 +349,85 @@ func (p *parser) stmt() ast.Stmt { } } else { // Match: for ([pre]; [cond]; [post]) body - p.expect(SEMICOLON) + p.expect(lexer.SEMICOLON) p.optionalNewlines() var cond ast.Expr - if p.tok != SEMICOLON { + if p.tok != lexer.SEMICOLON { cond = p.expr() } - p.expect(SEMICOLON) + p.expect(lexer.SEMICOLON) p.optionalNewlines() var post ast.Stmt - if p.tok != RPAREN { + if p.tok != lexer.RPAREN { post = p.simpleStmt() } - p.expect(RPAREN) + p.expect(lexer.RPAREN) p.optionalNewlines() bodyStart := p.pos body := p.loopStmts() s = &ast.ForStmt{Pre: pre, Cond: cond, Post: post, BodyStart: bodyStart, Body: body, Start: startPos, End: p.pos} } - case WHILE: + case lexer.WHILE: p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) cond := p.expr() - p.expect(RPAREN) + p.expect(lexer.RPAREN) p.optionalNewlines() bodyStart := p.pos body := p.loopStmts() s = &ast.WhileStmt{Cond: cond, BodyStart: bodyStart, Body: body, Start: startPos, End: p.pos} - case DO: + case lexer.DO: p.next() p.optionalNewlines() body := p.loopStmts() p.optionalNewlines() - p.expect(WHILE) - p.expect(LPAREN) + p.expect(lexer.WHILE) + p.expect(lexer.LPAREN) cond := p.expr() - p.expect(RPAREN) + p.expect(lexer.RPAREN) s = &ast.DoWhileStmt{Body: body, Cond: cond, Start: startPos, End: p.pos} - case BREAK: + case lexer.BREAK: if p.loopDepth == 0 { panic(p.errorf("break must be inside a loop body")) } p.next() s = &ast.BreakStmt{Start: startPos, End: p.pos} - case CONTINUE: + case lexer.CONTINUE: if p.loopDepth == 0 { panic(p.errorf("continue must be inside a loop body")) } p.next() s = &ast.ContinueStmt{Start: startPos, End: p.pos} - case NEXT: + case lexer.NEXT: if !p.inAction && p.funcName == "" { panic(p.errorf("next can't be inside BEGIN or END")) } p.next() s = &ast.NextStmt{Start: startPos, End: p.pos} - case NEXTFILE: + case lexer.NEXTFILE: if !p.inAction && p.funcName == "" { panic(p.errorf("nextfile can't be inside BEGIN or END")) } p.next() s = &ast.NextfileStmt{Start: startPos, End: p.pos} - case EXIT: + case lexer.EXIT: p.next() var status ast.Expr - if !p.matches(NEWLINE, SEMICOLON, RBRACE) { + if !p.matches(lexer.NEWLINE, lexer.SEMICOLON, lexer.RBRACE) { status = p.expr() } s = &ast.ExitStmt{Status: status, Start: startPos, End: p.pos} - case RETURN: + case lexer.RETURN: if p.funcName == "" { panic(p.errorf("return must be inside a function")) } p.next() var value ast.Expr - if !p.matches(NEWLINE, SEMICOLON, RBRACE) { + if !p.matches(lexer.NEWLINE, lexer.SEMICOLON, lexer.RBRACE) { value = p.expr() } s = &ast.ReturnStmt{Value: value, Start: startPos, End: p.pos} - case LBRACE: + case lexer.LBRACE: body := p.stmtsBrace() s = &ast.BlockStmt{Body: body, Start: startPos, End: p.pos} default: @@ -435,10 +435,10 @@ func (p *parser) stmt() ast.Stmt { } // Ensure statements are separated by ; or newline - if !p.matches(NEWLINE, SEMICOLON, RBRACE) && p.prevTok != NEWLINE && p.prevTok != SEMICOLON && p.prevTok != RBRACE { + if !p.matches(lexer.NEWLINE, lexer.SEMICOLON, lexer.RBRACE) && p.prevTok != lexer.NEWLINE && p.prevTok != lexer.SEMICOLON && p.prevTok != lexer.RBRACE { panic(p.errorf("expected ; or newline between statements")) } - for p.matches(NEWLINE, SEMICOLON) { + for p.matches(lexer.NEWLINE, lexer.SEMICOLON) { p.next() } return s @@ -464,11 +464,11 @@ func (p *parser) function() *ast.Function { } p.next() name, funcNamePos := p.expectName() - p.expect(LPAREN) + p.expect(lexer.LPAREN) first := true params := make([]string, 0, 7) // pre-allocate some to reduce allocations locals := make(map[string]bool, 7) - for p.tok != RPAREN { + for p.tok != lexer.RPAREN { if !first { p.commaNewlines() } @@ -480,11 +480,11 @@ func (p *parser) function() *ast.Function { if locals[param] { panic(p.errorf("duplicate parameter name %q", param)) } - p.expect(NAME) + p.expect(lexer.NAME) params = append(params, param) locals[param] = true } - p.expect(RPAREN) + p.expect(lexer.RPAREN) p.optionalNewlines() // Parse the body @@ -502,7 +502,7 @@ func (p *parser) function() *ast.Function { func (p *parser) exprList(parse func() ast.Expr) []ast.Expr { exprs := []ast.Expr{} first := true - for !p.matches(NEWLINE, SEMICOLON, RBRACE, RBRACKET, RPAREN, GREATER, PIPE, APPEND) { + for !p.matches(lexer.NEWLINE, lexer.SEMICOLON, lexer.RBRACE, lexer.RBRACKET, lexer.RPAREN, lexer.GREATER, lexer.PIPE, lexer.APPEND) { if !first { p.commaNewlines() } @@ -530,7 +530,7 @@ func (p *parser) getline() ast.Expr { // NOTE: getline is special, see https://github.com/benhoyt/goawk/pull/216 p.pendingGetlineLeft = nil left := p.cond() - if p.tok == PIPE { + if p.tok == lexer.PIPE { p.pendingGetlineLeft = left return p.cond() } @@ -546,7 +546,7 @@ func (p *parser) getline() ast.Expr { func (p *parser) _assign(higher func() ast.Expr) ast.Expr { leftPos := p.pos expr := higher() - if p.matches(ASSIGN, ADD_ASSIGN, DIV_ASSIGN, MOD_ASSIGN, MUL_ASSIGN, POW_ASSIGN, SUB_ASSIGN) { + if p.matches(lexer.ASSIGN, lexer.ADD_ASSIGN, lexer.DIV_ASSIGN, lexer.MOD_ASSIGN, lexer.MUL_ASSIGN, lexer.POW_ASSIGN, lexer.SUB_ASSIGN) { _, isNamedField := expr.(*ast.NamedFieldExpr) if isNamedField { panic(p.errorf("assigning @ expression not supported")) @@ -563,7 +563,7 @@ func (p *parser) _assign(higher func() ast.Expr) ast.Expr { binary, isBinary := expr.(*ast.BinaryExpr) if isBinary && ast.IsLValue(binary.Right) { switch binary.Op { - case AND, OR, MATCH, NOT_MATCH, EQUALS, NOT_EQUALS, LESS, LTE, GTE, GREATER: + case lexer.AND, lexer.OR, lexer.MATCH, lexer.NOT_MATCH, lexer.EQUALS, lexer.NOT_EQUALS, lexer.LESS, lexer.LTE, lexer.GTE, lexer.GREATER: assign := makeAssign(binary.Right, op, right) return &ast.BinaryExpr{Left: binary.Left, Op: binary.Op, Right: assign} } @@ -575,22 +575,22 @@ func (p *parser) _assign(higher func() ast.Expr) ast.Expr { return expr } -func makeAssign(left ast.Expr, op Token, right ast.Expr) ast.Expr { +func makeAssign(left ast.Expr, op lexer.Token, right ast.Expr) ast.Expr { switch op { - case ASSIGN: + case lexer.ASSIGN: return &ast.AssignExpr{Left: left, Right: right} - case ADD_ASSIGN: - op = ADD - case DIV_ASSIGN: - op = DIV - case MOD_ASSIGN: - op = MOD - case MUL_ASSIGN: - op = MUL - case POW_ASSIGN: - op = POW - case SUB_ASSIGN: - op = SUB + case lexer.ADD_ASSIGN: + op = lexer.ADD + case lexer.DIV_ASSIGN: + op = lexer.DIV + case lexer.MOD_ASSIGN: + op = lexer.MOD + case lexer.MUL_ASSIGN: + op = lexer.MUL + case lexer.POW_ASSIGN: + op = lexer.POW + case lexer.SUB_ASSIGN: + op = lexer.SUB } return &ast.AugAssignExpr{Left: left, Op: op, Right: right} } @@ -603,11 +603,11 @@ func (p *parser) printCond() ast.Expr { return p._cond(p.printOr) } func (p *parser) _cond(higher func() ast.Expr) ast.Expr { expr := higher() - if p.tok == QUESTION { + if p.tok == lexer.QUESTION { p.next() p.optionalNewlines() t := p.expr() - p.expect(COLON) + p.expect(lexer.COLON) p.optionalNewlines() f := p.expr() return &ast.CondExpr{Cond: expr, True: t, False: f} @@ -618,14 +618,14 @@ func (p *parser) _cond(higher func() ast.Expr) ast.Expr { // Parse an || or expression: // // and [OR NEWLINE* and] [OR NEWLINE* and] ... -func (p *parser) or() ast.Expr { return p.binaryLeft(p.and, true, OR) } -func (p *parser) printOr() ast.Expr { return p.binaryLeft(p.printAnd, true, OR) } +func (p *parser) or() ast.Expr { return p.binaryLeft(p.and, true, lexer.OR) } +func (p *parser) printOr() ast.Expr { return p.binaryLeft(p.printAnd, true, lexer.OR) } // Parse an && and expression: // // in [AND NEWLINE* in] [AND NEWLINE* in] ... -func (p *parser) and() ast.Expr { return p.binaryLeft(p.in, true, AND) } -func (p *parser) printAnd() ast.Expr { return p.binaryLeft(p.printIn, true, AND) } +func (p *parser) and() ast.Expr { return p.binaryLeft(p.in, true, lexer.AND) } +func (p *parser) printAnd() ast.Expr { return p.binaryLeft(p.printIn, true, lexer.AND) } // Parse an "in" expression: // @@ -635,7 +635,7 @@ func (p *parser) printIn() ast.Expr { return p._in(p.printMatch) } func (p *parser) _in(higher func() ast.Expr) ast.Expr { expr := higher() - for p.tok == IN { + for p.tok == lexer.IN { p.next() name, namePos := p.expectName() expr = &ast.InExpr{Index: []ast.Expr{expr}, Array: name, ArrayPos: namePos} @@ -651,7 +651,7 @@ func (p *parser) printMatch() ast.Expr { return p._match(p.printCompare) } func (p *parser) _match(higher func() ast.Expr) ast.Expr { expr := higher() - if p.matches(MATCH, NOT_MATCH) { + if p.matches(lexer.MATCH, lexer.NOT_MATCH) { op := p.tok p.next() right := p.regexStr(higher) // Not match() as these aren't associative @@ -663,10 +663,14 @@ func (p *parser) _match(higher func() ast.Expr) ast.Expr { // Parse a comparison expression: // // concat [EQUALS|NOT_EQUALS|LESS|LTE|GREATER|GTE concat] -func (p *parser) compare() ast.Expr { return p._compare(EQUALS, NOT_EQUALS, LESS, LTE, GTE, GREATER) } -func (p *parser) printCompare() ast.Expr { return p._compare(EQUALS, NOT_EQUALS, LESS, LTE, GTE) } +func (p *parser) compare() ast.Expr { + return p._compare(lexer.EQUALS, lexer.NOT_EQUALS, lexer.LESS, lexer.LTE, lexer.GTE, lexer.GREATER) +} +func (p *parser) printCompare() ast.Expr { + return p._compare(lexer.EQUALS, lexer.NOT_EQUALS, lexer.LESS, lexer.LTE, lexer.GTE) +} -func (p *parser) _compare(ops ...Token) ast.Expr { +func (p *parser) _compare(ops ...lexer.Token) ast.Expr { expr := p.concat() if p.matches(ops...) { op := p.tok @@ -679,36 +683,36 @@ func (p *parser) _compare(ops ...Token) ast.Expr { func (p *parser) concat() ast.Expr { expr := p.add() - for p.matches(DOLLAR, AT, NOT, NAME, NUMBER, STRING, LPAREN, INCR, DECR) || - p.tok >= FIRST_FUNC && p.tok <= LAST_FUNC { + for p.matches(lexer.DOLLAR, lexer.AT, lexer.NOT, lexer.NAME, lexer.NUMBER, lexer.STRING, lexer.LPAREN, lexer.INCR, lexer.DECR) || + p.tok >= lexer.FIRST_FUNC && p.tok <= lexer.LAST_FUNC { right := p.add() - expr = &ast.BinaryExpr{Left: expr, Op: CONCAT, Right: right} + expr = &ast.BinaryExpr{Left: expr, Op: lexer.CONCAT, Right: right} } return expr } func (p *parser) add() ast.Expr { - return p.binaryLeft(p.mul, false, ADD, SUB) + return p.binaryLeft(p.mul, false, lexer.ADD, lexer.SUB) } func (p *parser) mul() ast.Expr { - return p.binaryLeft(p.pow, false, MUL, DIV, MOD) + return p.binaryLeft(p.pow, false, lexer.MUL, lexer.DIV, lexer.MOD) } func (p *parser) pow() ast.Expr { // Note that pow (expr ^ expr) is right-associative expr := p.postIncr() - if p.tok == POW { + if p.tok == lexer.POW { p.next() right := p.pow() - return &ast.BinaryExpr{Left: expr, Op: POW, Right: right} + return &ast.BinaryExpr{Left: expr, Op: lexer.POW, Right: right} } return expr } func (p *parser) postIncr() ast.Expr { expr := p.primary() - if (p.tok == INCR || p.tok == DECR) && ast.IsLValue(expr) { + if (p.tok == lexer.INCR || p.tok == lexer.DECR) && ast.IsLValue(expr) { op := p.tok p.next() return &ast.IncrExpr{Expr: expr, Op: op} @@ -718,49 +722,49 @@ func (p *parser) postIncr() ast.Expr { func (p *parser) primary() ast.Expr { if p.pendingGetlineLeft != nil { - p.expect(PIPE) - p.expect(GETLINE) + p.expect(lexer.PIPE) + p.expect(lexer.GETLINE) left := p.pendingGetlineLeft p.pendingGetlineLeft = nil target := p.optionalLValue() return &ast.GetlineExpr{Command: left, Target: target} } switch p.tok { - case NUMBER: + case lexer.NUMBER: // AWK allows forms like "1.5e", but ParseFloat doesn't s := strings.TrimRight(p.val, "eE") n, _ := strconv.ParseFloat(s, 64) p.next() return &ast.NumExpr{Value: n} - case STRING: + case lexer.STRING: s := p.val p.next() return &ast.StrExpr{Value: s} - case DIV, DIV_ASSIGN: + case lexer.DIV, lexer.DIV_ASSIGN: // If we get to DIV or DIV_ASSIGN as a primary expression, // it's actually a regex. regex := p.nextRegex() return &ast.RegExpr{Regex: regex} - case DOLLAR: + case lexer.DOLLAR: p.next() var expr ast.Expr = &ast.FieldExpr{Index: p.primary()} // Post-increment operators have lower precedence than primary // expressions by default, except for field expressions with // post-increments (e.g., $$1++ = $($1++), NOT $($1)++). - if p.tok == INCR || p.tok == DECR { + if p.tok == lexer.INCR || p.tok == lexer.DECR { op := p.tok p.next() expr = &ast.IncrExpr{Expr: expr, Op: op} } return expr - case AT: + case lexer.AT: p.next() return &ast.NamedFieldExpr{Field: p.primary()} - case NOT, ADD, SUB: + case lexer.NOT, lexer.ADD, lexer.SUB: op := p.tok p.next() return &ast.UnaryExpr{Op: op, Value: p.pow()} - case INCR, DECR: + case lexer.INCR, lexer.DECR: op := p.tok p.next() exprPos := p.pos @@ -769,25 +773,25 @@ func (p *parser) primary() ast.Expr { panic(ast.PosErrorf(exprPos, "expected lvalue after %s", op)) } return &ast.IncrExpr{Expr: expr, Op: op, Pre: true} - case NAME: + case lexer.NAME: name, namePos := p.expectName() - if p.tok == LBRACKET { + if p.tok == lexer.LBRACKET { // a[x] or a[x, y] array index expression p.next() index := p.exprList(p.expr) if len(index) == 0 { panic(p.errorf("expected expression instead of ]")) } - p.expect(RBRACKET) + p.expect(lexer.RBRACKET) return &ast.IndexExpr{Array: name, ArrayPos: namePos, Index: index} - } else if p.tok == LPAREN && !p.lexer.HadSpace() { + } else if p.tok == lexer.LPAREN && !p.lexer.HadSpace() { // Grammar requires no space between function name and // left paren for user function calls, hence the funky // lexer.HadSpace() method. return p.userCall(name, namePos) } return &ast.VarExpr{Name: name, Pos: namePos} - case LPAREN: + case lexer.LPAREN: parenPos := p.pos p.next() exprs := p.exprList(p.expr) @@ -795,12 +799,12 @@ func (p *parser) primary() ast.Expr { case 0: panic(p.errorf("expected expression, not %s", p.tok)) case 1: - p.expect(RPAREN) + p.expect(lexer.RPAREN) return &ast.GroupingExpr{Expr: exprs[0]} default: // Multi-dimensional array "in" requires parens around index - p.expect(RPAREN) - if p.tok == IN { + p.expect(lexer.RPAREN) + if p.tok == lexer.IN { p.next() name, namePos := p.expectName() return &ast.InExpr{Index: exprs, Array: name, ArrayPos: namePos} @@ -808,11 +812,11 @@ func (p *parser) primary() ast.Expr { // MultiExpr is used as a pseudo-expression for print[f] parsing. return p.multiExpr(exprs, parenPos) } - case GETLINE: + case lexer.GETLINE: p.next() target := p.optionalLValue() var file ast.Expr - if p.tok == LESS { + if p.tok == lexer.LESS { p.next() file = p.primary() } @@ -821,15 +825,15 @@ func (p *parser) primary() ast.Expr { // could unify these but several of them have special handling // (array/lvalue/regex params, optional arguments, and so on). // Doing it this way means we can check more at parse time. - case F_SUB, F_GSUB: + case lexer.F_SUB, lexer.F_GSUB: op := p.tok p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) regex := p.regexStr(p.expr) p.commaNewlines() repl := p.expr() args := []ast.Expr{regex, repl} - if p.tok == COMMA { + if p.tok == lexer.COMMA { p.commaNewlines() inPos := p.pos in := p.expr() @@ -838,104 +842,104 @@ func (p *parser) primary() ast.Expr { } args = append(args, in) } - p.expect(RPAREN) + p.expect(lexer.RPAREN) return &ast.CallExpr{Func: op, Args: args} - case F_SPLIT: + case lexer.F_SPLIT: p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) str := p.expr() p.commaNewlines() name, namePos := p.expectName() args := []ast.Expr{str, &ast.VarExpr{Name: name, Pos: namePos}} - if p.tok == COMMA { + if p.tok == lexer.COMMA { p.commaNewlines() args = append(args, p.regexStr(p.expr)) } - p.expect(RPAREN) - return &ast.CallExpr{Func: F_SPLIT, Args: args} - case F_MATCH: + p.expect(lexer.RPAREN) + return &ast.CallExpr{Func: lexer.F_SPLIT, Args: args} + case lexer.F_MATCH: p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) str := p.expr() p.commaNewlines() regex := p.regexStr(p.expr) - p.expect(RPAREN) - return &ast.CallExpr{Func: F_MATCH, Args: []ast.Expr{str, regex}} - case F_RAND: + p.expect(lexer.RPAREN) + return &ast.CallExpr{Func: lexer.F_MATCH, Args: []ast.Expr{str, regex}} + case lexer.F_RAND: p.next() - p.expect(LPAREN) - p.expect(RPAREN) - return &ast.CallExpr{Func: F_RAND} - case F_SRAND: + p.expect(lexer.LPAREN) + p.expect(lexer.RPAREN) + return &ast.CallExpr{Func: lexer.F_RAND} + case lexer.F_SRAND: p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) var args []ast.Expr - if p.tok != RPAREN { + if p.tok != lexer.RPAREN { args = append(args, p.expr()) } - p.expect(RPAREN) - return &ast.CallExpr{Func: F_SRAND, Args: args} - case F_LENGTH: + p.expect(lexer.RPAREN) + return &ast.CallExpr{Func: lexer.F_SRAND, Args: args} + case lexer.F_LENGTH: p.next() var args []ast.Expr // AWK quirk: "length" is allowed to be called without parens - if p.tok == LPAREN { + if p.tok == lexer.LPAREN { p.next() - if p.tok != RPAREN { + if p.tok != lexer.RPAREN { args = append(args, p.expr()) } - p.expect(RPAREN) + p.expect(lexer.RPAREN) } - return &ast.CallExpr{Func: F_LENGTH, Args: args} - case F_SUBSTR: + return &ast.CallExpr{Func: lexer.F_LENGTH, Args: args} + case lexer.F_SUBSTR: p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) str := p.expr() p.commaNewlines() start := p.expr() args := []ast.Expr{str, start} - if p.tok == COMMA { + if p.tok == lexer.COMMA { p.commaNewlines() args = append(args, p.expr()) } - p.expect(RPAREN) - return &ast.CallExpr{Func: F_SUBSTR, Args: args} - case F_SPRINTF: + p.expect(lexer.RPAREN) + return &ast.CallExpr{Func: lexer.F_SUBSTR, Args: args} + case lexer.F_SPRINTF: p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) args := []ast.Expr{p.expr()} - for p.tok == COMMA { + for p.tok == lexer.COMMA { p.commaNewlines() args = append(args, p.expr()) } - p.expect(RPAREN) - return &ast.CallExpr{Func: F_SPRINTF, Args: args} - case F_FFLUSH: + p.expect(lexer.RPAREN) + return &ast.CallExpr{Func: lexer.F_SPRINTF, Args: args} + case lexer.F_FFLUSH: p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) var args []ast.Expr - if p.tok != RPAREN { + if p.tok != lexer.RPAREN { args = append(args, p.expr()) } - p.expect(RPAREN) - return &ast.CallExpr{Func: F_FFLUSH, Args: args} - case F_COS, F_SIN, F_EXP, F_LOG, F_SQRT, F_INT, F_TOLOWER, F_TOUPPER, F_SYSTEM, F_CLOSE: + p.expect(lexer.RPAREN) + return &ast.CallExpr{Func: lexer.F_FFLUSH, Args: args} + case lexer.F_COS, lexer.F_SIN, lexer.F_EXP, lexer.F_LOG, lexer.F_SQRT, lexer.F_INT, lexer.F_TOLOWER, lexer.F_TOUPPER, lexer.F_SYSTEM, lexer.F_CLOSE: // Simple 1-argument functions op := p.tok p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) arg := p.expr() - p.expect(RPAREN) + p.expect(lexer.RPAREN) return &ast.CallExpr{Func: op, Args: []ast.Expr{arg}} - case F_ATAN2, F_INDEX: + case lexer.F_ATAN2, lexer.F_INDEX: // Simple 2-argument functions op := p.tok p.next() - p.expect(LPAREN) + p.expect(lexer.LPAREN) arg1 := p.expr() p.commaNewlines() arg2 := p.expr() - p.expect(RPAREN) + p.expect(lexer.RPAREN) return &ast.CallExpr{Func: op, Args: []ast.Expr{arg1, arg2}} default: panic(p.errorf("expected expression instead of %s", p.tok)) @@ -945,24 +949,24 @@ func (p *parser) primary() ast.Expr { // Parse an optional lvalue func (p *parser) optionalLValue() ast.Expr { switch p.tok { - case NAME: + case lexer.NAME: if p.lexer.PeekByte() == '(' { // User function call, e.g., foo() not lvalue. return nil } name, namePos := p.expectName() - if p.tok == LBRACKET { + if p.tok == lexer.LBRACKET { // a[x] or a[x, y] array index expression p.next() index := p.exprList(p.expr) if len(index) == 0 { panic(p.errorf("expected expression instead of ]")) } - p.expect(RBRACKET) + p.expect(lexer.RBRACKET) return &ast.IndexExpr{Array: name, ArrayPos: namePos, Index: index} } return &ast.VarExpr{Name: name, Pos: namePos} - case DOLLAR: + case lexer.DOLLAR: p.next() return &ast.FieldExpr{Index: p.primary()} default: @@ -974,7 +978,7 @@ func (p *parser) optionalLValue() ast.Expr { // // REGEX | expr func (p *parser) regexStr(parse func() ast.Expr) ast.Expr { - if p.matches(DIV, DIV_ASSIGN) { + if p.matches(lexer.DIV, lexer.DIV_ASSIGN) { regex := p.nextRegex() return &ast.StrExpr{Value: regex, Regex: true} } @@ -985,7 +989,7 @@ func (p *parser) regexStr(parse func() ast.Expr) ast.Expr { // operator if allowNewline is true. // // parse [op parse] [op parse] ... -func (p *parser) binaryLeft(higher func() ast.Expr, allowNewline bool, ops ...Token) ast.Expr { +func (p *parser) binaryLeft(higher func() ast.Expr, allowNewline bool, ops ...lexer.Token) ast.Expr { expr := higher() for p.matches(ops...) { op := p.tok @@ -1003,7 +1007,7 @@ func (p *parser) binaryLeft(higher func() ast.Expr, allowNewline bool, ops ...To // // COMMA NEWLINE* func (p *parser) commaNewlines() { - p.expect(COMMA) + p.expect(lexer.COMMA) p.optionalNewlines() } @@ -1011,7 +1015,7 @@ func (p *parser) commaNewlines() { // // [NEWLINE] [NEWLINE] ... func (p *parser) optionalNewlines() { - for p.tok == NEWLINE { + for p.tok == lexer.NEWLINE { p.next() } } @@ -1020,7 +1024,7 @@ func (p *parser) optionalNewlines() { func (p *parser) next() { p.prevTok = p.tok p.pos, p.tok, p.val = p.lexer.Scan() - if p.tok == ILLEGAL { + if p.tok == lexer.ILLEGAL { panic(p.errorf("%s", p.val)) } } @@ -1029,7 +1033,7 @@ func (p *parser) next() { // DIV_ASSIGN token). func (p *parser) nextRegex() string { p.pos, p.tok, p.val = p.lexer.ScanRegex() - if p.tok == ILLEGAL { + if p.tok == lexer.ILLEGAL { panic(p.errorf("%s", p.val)) } regex := p.val @@ -1042,7 +1046,7 @@ func (p *parser) nextRegex() string { } // Ensure current token is tok, and parse next token into p.tok. -func (p *parser) expect(tok Token) { +func (p *parser) expect(tok lexer.Token) { if p.tok != tok { panic(p.errorf("expected %s instead of %s", tok, p.tok)) } @@ -1050,15 +1054,15 @@ func (p *parser) expect(tok Token) { } // Ensure current token is a name, parse it, and return name and position. -func (p *parser) expectName() (string, Position) { +func (p *parser) expectName() (string, lexer.Position) { name, pos := p.val, p.pos - p.expect(NAME) + p.expect(lexer.NAME) return name, pos } // Return true iff current token matches one of the given operators, // but don't parse next token. -func (p *parser) matches(operators ...Token) bool { +func (p *parser) matches(operators ...lexer.Token) bool { for _, operator := range operators { if p.tok == operator { return true @@ -1075,11 +1079,11 @@ func (p *parser) errorf(format string, args ...any) error { // Parse call to a user-defined function (and record call site for // resolving later). -func (p *parser) userCall(name string, pos Position) *ast.UserCallExpr { - p.expect(LPAREN) +func (p *parser) userCall(name string, pos lexer.Position) *ast.UserCallExpr { + p.expect(lexer.LPAREN) args := []ast.Expr{} i := 0 - for !p.matches(NEWLINE, RPAREN) { + for !p.matches(lexer.NEWLINE, lexer.RPAREN) { if i > 0 { p.commaNewlines() } @@ -1087,13 +1091,13 @@ func (p *parser) userCall(name string, pos Position) *ast.UserCallExpr { args = append(args, arg) i++ } - p.expect(RPAREN) + p.expect(lexer.RPAREN) return &ast.UserCallExpr{Name: name, Args: args, Pos: pos} } // Record a "multi expression" (comma-separated pseudo-expression // used to allow commas around print/printf arguments). -func (p *parser) multiExpr(exprs []ast.Expr, pos Position) ast.Expr { +func (p *parser) multiExpr(exprs []ast.Expr, pos lexer.Position) ast.Expr { expr := &ast.MultiExpr{Exprs: exprs} p.multiExprs[expr] = pos return expr @@ -1110,7 +1114,7 @@ func (p *parser) checkMultiExprs() { return } // Show error on first comma-separated expression - min := Position{Line: 1000000000, Column: 1000000000} + min := lexer.Position{Line: 1000000000, Column: 1000000000} for _, pos := range p.multiExprs { if pos.Line < min.Line || pos.Line == min.Line && pos.Column < min.Column { min = pos