From fdeaf881f44fbccf478112bda2d091b20b65cc0f Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 21 Mar 2018 17:00:39 -0700 Subject: [PATCH] Make End() return the position after the node End() was previously only used to determine if a comment was within a Node, so it used the expedient definition of the position of the last token in the node. In the next patch it will be used for capturing substrings of the Blueprint file, so make it point to the character after the last token instead. Also add tests for it. Test: parser_test.go Change-Id: Icaff3915b41e251ef9d0aad5615021bf37406aee --- parser/ast.go | 27 ++++--- parser/parser.go | 3 + parser/parser_test.go | 160 +++++++++++++++++++++++++++++++++--------- 3 files changed, 147 insertions(+), 43 deletions(-) diff --git a/parser/ast.go b/parser/ast.go index 7f94efb..b5053bb 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -21,9 +21,9 @@ import ( ) type Node interface { - // Pos returns the position of the first token in the Expression + // Pos returns the position of the first token in the Node Pos() scanner.Position - // End returns the position of the beginning of the last token in the Expression + // End returns the position of the character after the last token in the Node End() scanner.Position } @@ -220,7 +220,7 @@ type Variable struct { } func (x *Variable) Pos() scanner.Position { return x.NamePos } -func (x *Variable) End() scanner.Position { return x.NamePos } +func (x *Variable) End() scanner.Position { return endPos(x.NamePos, len(x.Name)) } func (x *Variable) Copy() Expression { ret := *x @@ -244,7 +244,7 @@ type Map struct { } func (x *Map) Pos() scanner.Position { return x.LBracePos } -func (x *Map) End() scanner.Position { return x.RBracePos } +func (x *Map) End() scanner.Position { return endPos(x.RBracePos, 1) } func (x *Map) Copy() Expression { ret := *x @@ -302,7 +302,7 @@ type List struct { } func (x *List) Pos() scanner.Position { return x.LBracePos } -func (x *List) End() scanner.Position { return x.RBracePos } +func (x *List) End() scanner.Position { return endPos(x.RBracePos, 1) } func (x *List) Copy() Expression { ret := *x @@ -334,7 +334,7 @@ type String struct { } func (x *String) Pos() scanner.Position { return x.LiteralPos } -func (x *String) End() scanner.Position { return x.LiteralPos } +func (x *String) End() scanner.Position { return endPos(x.LiteralPos, len(x.Value)+2) } func (x *String) Copy() Expression { ret := *x @@ -356,10 +356,11 @@ func (x *String) Type() Type { type Int64 struct { LiteralPos scanner.Position Value int64 + Token string } func (x *Int64) Pos() scanner.Position { return x.LiteralPos } -func (x *Int64) End() scanner.Position { return x.LiteralPos } +func (x *Int64) End() scanner.Position { return endPos(x.LiteralPos, len(x.Token)) } func (x *Int64) Copy() Expression { ret := *x @@ -381,10 +382,11 @@ func (x *Int64) Type() Type { type Bool struct { LiteralPos scanner.Position Value bool + Token string } func (x *Bool) Pos() scanner.Position { return x.LiteralPos } -func (x *Bool) End() scanner.Position { return x.LiteralPos } +func (x *Bool) End() scanner.Position { return endPos(x.LiteralPos, len(x.Token)) } func (x *Bool) Copy() Expression { ret := *x @@ -422,7 +424,8 @@ func (c Comment) Pos() scanner.Position { func (c Comment) End() scanner.Position { pos := c.Slash for _, comment := range c.Comment { - pos.Offset += len(comment) + pos.Offset += len(comment) + 1 + pos.Column = len(comment) + 1 } pos.Line += len(c.Comment) - 1 return pos @@ -472,3 +475,9 @@ func (c Comment) Text() string { return string(buf) } + +func endPos(pos scanner.Position, n int) scanner.Position { + pos.Offset += n + pos.Column += n + return pos +} diff --git a/parser/parser.go b/parser/parser.go index 728afbe..e832e1a 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -360,6 +360,7 @@ func (p *parser) evaluateOperator(value1, value2 Expression, operator rune, v.Value += e2.(*String).Value case *Int64: v.Value += e2.(*Int64).Value + v.Token = "" case *List: v.Values = append(v.Values, e2.(*List).Values...) case *Map: @@ -469,6 +470,7 @@ func (p *parser) parseVariable() Expression { value = &Bool{ LiteralPos: p.scanner.Position, Value: text == "true", + Token: text, } default: if p.eval { @@ -528,6 +530,7 @@ func (p *parser) parseIntValue() *Int64 { value := &Int64{ LiteralPos: literalPos, Value: i, + Token: str, } p.accept(scanner.Int) return value diff --git a/parser/parser_test.go b/parser/parser_test.go index d740184..6377dc1 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -17,6 +17,8 @@ package parser import ( "bytes" "reflect" + "strconv" + "strings" "testing" "text/scanner" ) @@ -99,6 +101,7 @@ var validParseTestCases = []struct { Value: &Bool{ LiteralPos: mkpos(20, 3, 12), Value: true, + Token: "true", }, }, }, @@ -128,6 +131,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(17, 3, 9), Value: 4, + Token: "4", }, }, }, @@ -221,6 +225,7 @@ var validParseTestCases = []struct { Value: &Bool{ LiteralPos: mkpos(33, 4, 13), Value: true, + Token: "true", }, }, { @@ -239,6 +244,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(65, 6, 10), Value: 36, + Token: "36", }, }, }, @@ -273,6 +279,7 @@ var validParseTestCases = []struct { Value: &Bool{ LiteralPos: mkpos(60, 5, 12), Value: true, + Token: "true", }, }, }, @@ -350,6 +357,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(33, 4, 9), Value: 4, + Token: "4", }, }, }, @@ -378,6 +386,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(73, 9, 9), Value: -5, + Token: "-5", }, }, }, @@ -637,6 +646,7 @@ var validParseTestCases = []struct { &Int64{ LiteralPos: mkpos(9, 2, 9), Value: -4, + Token: "-4", }, &Operator{ OperatorPos: mkpos(17, 2, 17), @@ -649,10 +659,12 @@ var validParseTestCases = []struct { &Int64{ LiteralPos: mkpos(14, 2, 14), Value: -5, + Token: "-5", }, &Int64{ LiteralPos: mkpos(19, 2, 19), Value: 6, + Token: "6", }, }, }, @@ -669,6 +681,7 @@ var validParseTestCases = []struct { &Int64{ LiteralPos: mkpos(9, 2, 9), Value: -4, + Token: "-4", }, &Operator{ OperatorPos: mkpos(17, 2, 17), @@ -681,10 +694,12 @@ var validParseTestCases = []struct { &Int64{ LiteralPos: mkpos(14, 2, 14), Value: -5, + Token: "-5", }, &Int64{ LiteralPos: mkpos(19, 2, 19), Value: 6, + Token: "6", }, }, }, @@ -712,10 +727,12 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, OrigValue: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, Assigner: "=", Referenced: true, @@ -730,6 +747,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, OrigValue: &Variable{ @@ -738,6 +756,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, Assigner: "=", @@ -761,6 +780,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, &Variable{ @@ -772,6 +792,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -791,6 +812,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, &Variable{ @@ -802,6 +824,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -833,6 +856,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, &Variable{ @@ -844,6 +868,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -856,6 +881,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -883,6 +909,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, &Variable{ @@ -894,6 +921,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, }, @@ -912,6 +940,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, OrigValue: &Variable{ @@ -920,6 +949,7 @@ var validParseTestCases = []struct { Value: &Int64{ LiteralPos: mkpos(9, 2, 9), Value: 1000000, + Token: "1000000", }, }, Assigner: "+=", @@ -985,48 +1015,110 @@ var validParseTestCases = []struct { } func TestParseValidInput(t *testing.T) { - for _, testCase := range validParseTestCases { - r := bytes.NewBufferString(testCase.input) - file, errs := ParseAndEval("", r, NewScope(nil)) - if len(errs) != 0 { - t.Errorf("test case: %s", testCase.input) - t.Errorf("unexpected errors:") - for _, err := range errs { - t.Errorf(" %s", err) + for i, testCase := range validParseTestCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + r := bytes.NewBufferString(testCase.input) + file, errs := ParseAndEval("", r, NewScope(nil)) + if len(errs) != 0 { + t.Errorf("test case: %s", testCase.input) + t.Errorf("unexpected errors:") + for _, err := range errs { + t.Errorf(" %s", err) + } + t.FailNow() } - t.FailNow() - } - if len(file.Defs) == len(testCase.defs) { - for i := range file.Defs { - if !reflect.DeepEqual(file.Defs[i], testCase.defs[i]) { - t.Errorf("test case: %s", testCase.input) - t.Errorf("incorrect defintion %d:", i) - t.Errorf(" expected: %s", testCase.defs[i]) - t.Errorf(" got: %s", file.Defs[i]) + if len(file.Defs) == len(testCase.defs) { + for i := range file.Defs { + if !reflect.DeepEqual(file.Defs[i], testCase.defs[i]) { + t.Errorf("test case: %s", testCase.input) + t.Errorf("incorrect defintion %d:", i) + t.Errorf(" expected: %s", testCase.defs[i]) + t.Errorf(" got: %s", file.Defs[i]) + } } + } else { + t.Errorf("test case: %s", testCase.input) + t.Errorf("length mismatch, expected %d definitions, got %d", + len(testCase.defs), len(file.Defs)) } - } else { - t.Errorf("test case: %s", testCase.input) - t.Errorf("length mismatch, expected %d definitions, got %d", - len(testCase.defs), len(file.Defs)) - } - if len(file.Comments) == len(testCase.comments) { - for i := range file.Comments { - if !reflect.DeepEqual(file.Comments[i], testCase.comments[i]) { - t.Errorf("test case: %s", testCase.input) - t.Errorf("incorrect comment %d:", i) - t.Errorf(" expected: %s", testCase.comments[i]) - t.Errorf(" got: %s", file.Comments[i]) + if len(file.Comments) == len(testCase.comments) { + for i := range file.Comments { + if !reflect.DeepEqual(file.Comments[i], testCase.comments[i]) { + t.Errorf("test case: %s", testCase.input) + t.Errorf("incorrect comment %d:", i) + t.Errorf(" expected: %s", testCase.comments[i]) + t.Errorf(" got: %s", file.Comments[i]) + } } + } else { + t.Errorf("test case: %s", testCase.input) + t.Errorf("length mismatch, expected %d comments, got %d", + len(testCase.comments), len(file.Comments)) } - } else { - t.Errorf("test case: %s", testCase.input) - t.Errorf("length mismatch, expected %d comments, got %d", - len(testCase.comments), len(file.Comments)) - } + }) } } // TODO: Test error strings + +func TestParserEndPos(t *testing.T) { + in := ` + module { + string: "string", + stringexp: "string1" + "string2", + int: -1, + intexp: -1 + 2, + list: ["a", "b"], + listexp: ["c"] + ["d"], + multilinelist: [ + "e", + "f", + ], + map: { + prop: "abc", + }, + } + ` + + // Strip each line to make it easier to compute the previous "," from each property + lines := strings.Split(in, "\n") + for i := range lines { + lines[i] = strings.TrimSpace(lines[i]) + } + in = strings.Join(lines, "\n") + + r := bytes.NewBufferString(in) + + file, errs := ParseAndEval("", r, NewScope(nil)) + if len(errs) != 0 { + t.Errorf("unexpected errors:") + for _, err := range errs { + t.Errorf(" %s", err) + } + t.FailNow() + } + + mod := file.Defs[0].(*Module) + modEnd := mkpos(len(in)-1, len(lines)-1, 2) + if mod.End() != modEnd { + t.Errorf("expected mod.End() %s, got %s", modEnd, mod.End()) + } + + nextPos := make([]scanner.Position, len(mod.Properties)) + for i := 0; i < len(mod.Properties)-1; i++ { + nextPos[i] = mod.Properties[i+1].Pos() + } + nextPos[len(mod.Properties)-1] = mod.RBracePos + + for i, cur := range mod.Properties { + endOffset := nextPos[i].Offset - len(",\n") + endLine := nextPos[i].Line - 1 + endColumn := len(lines[endLine-1]) // scanner.Position.Line is starts at 1 + endPos := mkpos(endOffset, endLine, endColumn) + if cur.End() != endPos { + t.Errorf("expected property %s End() %s@%d, got %s@%d", cur.Name, endPos, endPos.Offset, cur.End(), cur.End().Offset) + } + } +} -- GitLab