diff --git a/parser/ast.go b/parser/ast.go index 7f94efb3bb12f01ae5391f14a743eadccd1a99be..b5053bbb8098db2f8552c8169720a7cced2f4814 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -21,9 +21,9 @@ import ( ) type Node interface { - // Pos returns the position of the first token in the Expression + // Pos returns the position of the first token in the Node Pos() scanner.Position - // End returns the position of the beginning of the last token in the Expression + // End returns the position of the character after the last token in the Node End() scanner.Position } @@ -220,7 +220,7 @@ type Variable struct { } func (x *Variable) Pos() scanner.Position { return x.NamePos } -func (x *Variable) End() scanner.Position { return x.NamePos } +func (x *Variable) End() scanner.Position { return endPos(x.NamePos, len(x.Name)) } func (x *Variable) Copy() Expression { ret := *x @@ -244,7 +244,7 @@ type Map struct { } func (x *Map) Pos() scanner.Position { return x.LBracePos } -func (x *Map) End() scanner.Position { return x.RBracePos } +func (x *Map) End() scanner.Position { return endPos(x.RBracePos, 1) } func (x *Map) Copy() Expression { ret := *x @@ -302,7 +302,7 @@ type List struct { } func (x *List) Pos() scanner.Position { return x.LBracePos } -func (x *List) End() scanner.Position { return x.RBracePos } +func (x *List) End() scanner.Position { return endPos(x.RBracePos, 1) } func (x *List) Copy() Expression { ret := *x @@ -334,7 +334,7 @@ type String struct { } func (x *String) Pos() scanner.Position { return x.LiteralPos } -func (x *String) End() scanner.Position { return x.LiteralPos } +func (x *String) End() scanner.Position { return endPos(x.LiteralPos, len(x.Value)+2) } func (x *String) Copy() Expression { ret := *x @@ -356,10 +356,11 @@ func (x *String) Type() Type { type Int64 struct { LiteralPos scanner.Position Value int64 + Token string } func (x *Int64) Pos() scanner.Position { return x.LiteralPos } -func (x *Int64) End() scanner.Position { return x.LiteralPos } +func (x *Int64) End() scanner.Position { return endPos(x.LiteralPos, len(x.Token)) } func (x *Int64) Copy() Expression { ret := *x @@ -381,10 +382,11 @@ func (x *Int64) Type() Type { type Bool struct { LiteralPos scanner.Position Value bool + Token string } func (x *Bool) Pos() scanner.Position { return x.LiteralPos } -func (x *Bool) End() scanner.Position { return x.LiteralPos } +func (x *Bool) End() scanner.Position { return endPos(x.LiteralPos, len(x.Token)) } func (x *Bool) Copy() Expression { ret := *x @@ -422,7 +424,8 @@ func (c Comment) Pos() scanner.Position { func (c Comment) End() scanner.Position { pos := c.Slash for _, comment := range c.Comment { - pos.Offset += len(comment) + pos.Offset += len(comment) + 1 + pos.Column = len(comment) + 1 } pos.Line += len(c.Comment) - 1 return pos @@ -472,3 +475,9 @@ func (c Comment) Text() string { return string(buf) } + +func endPos(pos scanner.Position, n int) scanner.Position { + pos.Offset += n + pos.Column += n + return pos +} diff --git a/parser/parser.go b/parser/parser.go index 728afbe838f451a101f01cb4a06b2e704db36252..e832e1ac856a6a42188fe0fa0b756d421aef164d 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -360,6 +360,7 @@ func (p *parser) evaluateOperator(value1, value2 Expression, operator rune, v.Value += e2.(*String).Value case *Int64: v.Value += e2.(*Int64).Value + v.Token = "" case *List: v.Values = append(v.Values, e2.(*List).Values...) case *Map: @@ -469,6 +470,7 @@ func (p *parser) parseVariable() Expression { value = &Bool{ LiteralPos: p.scanner.Position, Value: text == "true", + Token: text, } default: if p.eval { @@ -528,6 +530,7 @@ func (p *parser) parseIntValue() *Int64 { value := &Int64{ LiteralPos: literalPos, Value: i, + Token: str, } p.accept(scanner.Int) return value diff --git a/parser/parser_test.go b/parser/parser_test.go index d740184d289f008a4463c777a4f4ed047a9f912d..6377dc103c3b878d5e56cdffb251865f90e1935c 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -17,6 +17,8 @@ package parser import ( "bytes" "reflect" + "strconv" + "strings" "testing" "text/scanner" ) @@ -99,6 +101,7 @@ var validParseTestCases = []struct { Value: &Bool{ LiteralPos: mkpos(20, 3, 12), Value: true, + Token: "true", }, }, }, @@ -128,6 +131,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(17, 3, 9), Value: 4, + Token: "4", }, }, }, @@ -221,6 +225,7 @@ var validParseTestCases = []struct { Value: &Bool{ LiteralPos: mkpos(33, 4, 13), Value: true, + Token: "true", }, }, { @@ -239,6 +244,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(65, 6, 10), Value: 36, + Token: "36", }, }, }, @@ -273,6 +279,7 @@ var validParseTestCases = []struct { Value: &Bool{ LiteralPos: mkpos(60, 5, 12), Value: true, + Token: "true", }, }, }, @@ -350,6 +357,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(33, 4, 9), Value: 4, + Token: "4", }, }, }, @@ -378,6 +386,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(73, 9, 9), Value: -5, + Token: "-5", }, }, }, @@ -637,6 +646,7 @@ var validParseTestCases = []struct { &Int64{ LiteralPos: mkpos(9, 2, 9), Value: -4, + Token: "-4", }, &Operator{ OperatorPos: mkpos(17, 2, 17), @@ -649,10 +659,12 @@ var validParseTestCases = []struct { &Int64{ LiteralPos: mkpos(14, 2, 14), Value: -5, + Token: "-5", }, &Int64{ LiteralPos: mkpos(19, 2, 19), Value: 6, + Token: "6", }, }, }, @@ -669,6 +681,7 @@ var validParseTestCases = []struct { &Int64{ LiteralPos: mkpos(9, 2, 9), Value: -4, + Token: "-4", }, &Operator{ OperatorPos: mkpos(17, 2, 17), @@ -681,10 +694,12 @@ var validParseTestCases = []struct { &Int64{ LiteralPos: mkpos(14, 2, 14), Value: -5, + Token: "-5", }, &Int64{ LiteralPos: mkpos(19, 2, 19), Value: 6, + Token: "6", }, }, }, @@ -712,10 +727,12 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, OrigValue: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, Assigner: "=", Referenced: true, @@ -730,6 +747,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, OrigValue: &Variable{ @@ -738,6 +756,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, Assigner: "=", @@ -761,6 +780,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, &Variable{ @@ -772,6 +792,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -791,6 +812,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, &Variable{ @@ -802,6 +824,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -833,6 +856,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, &Variable{ @@ -844,6 +868,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -856,6 +881,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -883,6 +909,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, &Variable{ @@ -894,6 +921,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -912,6 +940,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, OrigValue: &Variable{ @@ -920,6 +949,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, Assigner: "+=", @@ -985,48 +1015,110 @@ var validParseTestCases = []struct { } func TestParseValidInput(t *testing.T) { - for _, testCase := range validParseTestCases { - r := bytes.NewBufferString(testCase.input) - file, errs := ParseAndEval("", r, NewScope(nil)) - if len(errs) != 0 { - t.Errorf("test case: %s", testCase.input) - t.Errorf("unexpected errors:") - for _, err := range errs { - t.Errorf(" %s", err) + for i, testCase := range validParseTestCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + r := bytes.NewBufferString(testCase.input) + file, errs := ParseAndEval("", r, NewScope(nil)) + if len(errs) != 0 { + t.Errorf("test case: %s", testCase.input) + t.Errorf("unexpected errors:") + for _, err := range errs { + t.Errorf(" %s", err) + } + t.FailNow() } - t.FailNow() - } - if len(file.Defs) == len(testCase.defs) { - for i := range file.Defs { - if !reflect.DeepEqual(file.Defs[i], testCase.defs[i]) { - t.Errorf("test case: %s", testCase.input) - t.Errorf("incorrect defintion %d:", i) - t.Errorf(" expected: %s", testCase.defs[i]) - t.Errorf(" got: %s", file.Defs[i]) + if len(file.Defs) == len(testCase.defs) { + for i := range file.Defs { + if !reflect.DeepEqual(file.Defs[i], testCase.defs[i]) { + t.Errorf("test case: %s", testCase.input) + t.Errorf("incorrect defintion %d:", i) + t.Errorf(" expected: %s", testCase.defs[i]) + t.Errorf(" got: %s", file.Defs[i]) + } } + } else { + t.Errorf("test case: %s", testCase.input) + t.Errorf("length mismatch, expected %d definitions, got %d", + len(testCase.defs), len(file.Defs)) } - } else { - t.Errorf("test case: %s", testCase.input) - t.Errorf("length mismatch, expected %d definitions, got %d", - len(testCase.defs), len(file.Defs)) - } - if len(file.Comments) == len(testCase.comments) { - for i := range file.Comments { - if !reflect.DeepEqual(file.Comments[i], testCase.comments[i]) { - t.Errorf("test case: %s", testCase.input) - t.Errorf("incorrect comment %d:", i) - t.Errorf(" expected: %s", testCase.comments[i]) - t.Errorf(" got: %s", file.Comments[i]) + if len(file.Comments) == len(testCase.comments) { + for i := range file.Comments { + if !reflect.DeepEqual(file.Comments[i], testCase.comments[i]) { + t.Errorf("test case: %s", testCase.input) + t.Errorf("incorrect comment %d:", i) + t.Errorf(" expected: %s", testCase.comments[i]) + t.Errorf(" got: %s", file.Comments[i]) + } } + } else { + t.Errorf("test case: %s", testCase.input) + t.Errorf("length mismatch, expected %d comments, got %d", + len(testCase.comments), len(file.Comments)) } - } else { - t.Errorf("test case: %s", testCase.input) - t.Errorf("length mismatch, expected %d comments, got %d", - len(testCase.comments), len(file.Comments)) - } + }) } } // TODO: Test error strings + +func TestParserEndPos(t *testing.T) { + in := ` + module { + string: "string", + stringexp: "string1" + "string2", + int: -1, + intexp: -1 + 2, + list: ["a", "b"], + listexp: ["c"] + ["d"], + multilinelist: [ + "e", + "f", + ], + map: { + prop: "abc", + }, + } + ` + + // Strip each line to make it easier to compute the previous "," from each property + lines := strings.Split(in, "\n") + for i := range lines { + lines[i] = strings.TrimSpace(lines[i]) + } + in = strings.Join(lines, "\n") + + r := bytes.NewBufferString(in) + + file, errs := ParseAndEval("", r, NewScope(nil)) + if len(errs) != 0 { + t.Errorf("unexpected errors:") + for _, err := range errs { + t.Errorf(" %s", err) + } + t.FailNow() + } + + mod := file.Defs[0].(*Module) + modEnd := mkpos(len(in)-1, len(lines)-1, 2) + if mod.End() != modEnd { + t.Errorf("expected mod.End() %s, got %s", modEnd, mod.End()) + } + + nextPos := make([]scanner.Position, len(mod.Properties)) + for i := 0; i < len(mod.Properties)-1; i++ { + nextPos[i] = mod.Properties[i+1].Pos() + } + nextPos[len(mod.Properties)-1] = mod.RBracePos + + for i, cur := range mod.Properties { + endOffset := nextPos[i].Offset - len(",\n") + endLine := nextPos[i].Line - 1 + endColumn := len(lines[endLine-1]) // scanner.Position.Line is starts at 1 + endPos := mkpos(endOffset, endLine, endColumn) + if cur.End() != endPos { + t.Errorf("expected property %s End() %s@%d, got %s@%d", cur.Name, endPos, endPos.Offset, cur.End(), cur.End().Offset) + } + } +}