7 "github.com/hashicorp/hil/ast"
8 "github.com/hashicorp/hil/scanner"
11 func Parse(ch <-chan *scanner.Token) (ast.Node, error) {
12 peeker := scanner.NewPeeker(ch)
13 parser := &parser{peeker}
14 output, err := parser.ParseTopLevel()
20 peeker *scanner.Peeker
23 func (p *parser) ParseTopLevel() (ast.Node, error) {
24 return p.parseInterpolationSeq(false)
27 func (p *parser) ParseQuoted() (ast.Node, error) {
28 return p.parseInterpolationSeq(true)
31 // parseInterpolationSeq parses either the top-level sequence of literals
32 // and interpolation expressions or a similar sequence within a quoted
33 // string inside an interpolation expression. The latter case is requested
34 // by setting 'quoted' to true.
35 func (p *parser) parseInterpolationSeq(quoted bool) (ast.Node, error) {
36 literalType := scanner.LITERAL
37 endType := scanner.EOF
39 // exceptions for quoted sequences
40 literalType = scanner.STRING
41 endType = scanner.CQUOTE
44 startPos := p.peeker.Peek().Pos
47 tok := p.peeker.Read()
48 if tok.Type != scanner.OQUOTE {
49 return nil, ExpectationError("open quote", tok)
55 tok := p.peeker.Read()
57 if tok.Type == endType {
63 val, err := p.parseStringToken(tok)
67 exprs = append(exprs, &ast.LiteralNode{
69 Typex: ast.TypeString,
73 expr, err := p.ParseInterpolation()
77 exprs = append(exprs, expr)
79 return nil, ExpectationError(`"${"`, tok)
84 // If we have no parts at all then the input must've
85 // been an empty string.
86 exprs = append(exprs, &ast.LiteralNode{
88 Typex: ast.TypeString,
93 // As a special case, if our "Output" contains only one expression
94 // and it's a literal string then we'll hoist it up to be our
95 // direct return value, so callers can easily recognize a string
96 // that has no interpolations at all.
98 if lit, ok := exprs[0].(*ast.LiteralNode); ok {
99 if lit.Typex == ast.TypeString {
111 // parseStringToken takes a token of either LITERAL or STRING type and
112 // returns the interpreted string, after processing any relevant
114 func (p *parser) parseStringToken(tok *scanner.Token) (string, error) {
117 case scanner.LITERAL:
122 panic("unsupported string token type")
125 raw := []byte(tok.Content)
126 buf := make([]byte, 0, len(raw))
128 for i := 0; i < len(raw); i++ {
130 more := len(raw) > (i + 1)
133 if more && raw[i+1] == '$' {
134 // skip over the second dollar sign
137 } else if backslashes && b == '\\' {
141 Column: tok.Pos.Column + utf8.RuneCount(raw[:i]),
144 `unfinished backslash escape sequence`,
147 escapeType := raw[i+1]
150 // skip over the second slash
161 Column: tok.Pos.Column + utf8.RuneCount(raw[:i]),
164 `invalid backslash escape sequence`,
172 return string(buf), nil
175 func (p *parser) ParseInterpolation() (ast.Node, error) {
176 // By the time we're called, we're already "inside" the ${ sequence
177 // because the caller consumed the ${ token.
179 expr, err := p.ParseExpression()
184 err = p.requireTokenType(scanner.END, `"}"`)
192 func (p *parser) ParseExpression() (ast.Node, error) {
193 return p.parseTernaryCond()
196 func (p *parser) parseTernaryCond() (ast.Node, error) {
197 // The ternary condition operator (.. ? .. : ..) behaves somewhat
198 // like a binary operator except that the "operator" is itself
199 // an expression enclosed in two punctuation characters.
200 // The middle expression is parsed as if the ? and : symbols
201 // were parentheses. The "rhs" (the "false expression") is then
202 // treated right-associatively so it behaves similarly to the
203 // middle in terms of precedence.
205 startPos := p.peeker.Peek().Pos
207 var cond, trueExpr, falseExpr ast.Node
210 cond, err = p.parseBinaryOps(binaryOps)
215 next := p.peeker.Peek()
216 if next.Type != scanner.QUESTION {
220 p.peeker.Read() // eat question mark
222 trueExpr, err = p.ParseExpression()
227 colon := p.peeker.Read()
228 if colon.Type != scanner.COLON {
229 return nil, ExpectationError(":", colon)
232 falseExpr, err = p.ParseExpression()
237 return &ast.Conditional{
240 FalseExpr: falseExpr,
245 // parseBinaryOps calls itself recursively to work through all of the
246 // operator precedence groups, and then eventually calls ParseExpressionTerm
248 func (p *parser) parseBinaryOps(ops []map[scanner.TokenType]ast.ArithmeticOp) (ast.Node, error) {
250 // We've run out of operators, so now we'll just try to parse a term.
251 return p.ParseExpressionTerm()
257 startPos := p.peeker.Peek().Pos
259 var lhs, rhs ast.Node
260 operator := ast.ArithmeticOpInvalid
263 // parse a term that might be the first operand of a binary
264 // expression or it might just be a standalone term, but
265 // we won't know until we've parsed it and can look ahead
266 // to see if there's an operator token.
267 lhs, err = p.parseBinaryOps(remaining)
272 // We'll keep eating up arithmetic operators until we run
273 // out, so that operators with the same precedence will combine in a
274 // left-associative manner:
275 // a+b+c => (a+b)+c, not a+(b+c)
277 // Should we later want to have right-associative operators, a way
278 // to achieve that would be to call back up to ParseExpression here
279 // instead of iteratively parsing only the remaining operators.
281 next := p.peeker.Peek()
282 var newOperator ast.ArithmeticOp
284 if newOperator, ok = thisLevel[next.Type]; !ok {
288 // Are we extending an expression started on
289 // the previous iteration?
290 if operator != ast.ArithmeticOpInvalid {
291 lhs = &ast.Arithmetic{
293 Exprs: []ast.Node{lhs, rhs},
298 operator = newOperator
299 p.peeker.Read() // eat operator token
300 rhs, err = p.parseBinaryOps(remaining)
306 if operator != ast.ArithmeticOpInvalid {
307 return &ast.Arithmetic{
309 Exprs: []ast.Node{lhs, rhs},
317 func (p *parser) ParseExpressionTerm() (ast.Node, error) {
319 next := p.peeker.Peek()
325 expr, err := p.ParseExpression()
329 err = p.requireTokenType(scanner.CPAREN, `")"`)
333 return p.ParseQuoted()
335 case scanner.INTEGER:
336 tok := p.peeker.Read()
337 val, err := strconv.Atoi(tok.Content)
339 return nil, TokenErrorf(tok, "invalid integer: %s", err)
341 return &ast.LiteralNode{
348 tok := p.peeker.Read()
349 val, err := strconv.ParseFloat(tok.Content, 64)
351 return nil, TokenErrorf(tok, "invalid float: %s", err)
353 return &ast.LiteralNode{
355 Typex: ast.TypeFloat,
360 tok := p.peeker.Read()
361 // the scanner guarantees that tok.Content is either "true" or "false"
363 if tok.Content[0] == 't' {
368 return &ast.LiteralNode{
375 opTok := p.peeker.Read()
376 // important to use ParseExpressionTerm rather than ParseExpression
377 // here, otherwise we can capture a following binary expression into
379 // e.g. -46+5 should parse as (0-46)+5, not 0-(46+5)
380 operand, err := p.ParseExpressionTerm()
384 // The AST currently represents negative numbers as
385 // a binary subtraction of the number from zero.
386 return &ast.Arithmetic{
387 Op: ast.ArithmeticOpSub,
400 opTok := p.peeker.Read()
401 // important to use ParseExpressionTerm rather than ParseExpression
402 // here, otherwise we can capture a following binary expression into
404 operand, err := p.ParseExpressionTerm()
408 // The AST currently represents binary negation as an equality
409 // test with "false".
410 return &ast.Arithmetic{
411 Op: ast.ArithmeticOpEqual,
423 case scanner.IDENTIFIER:
424 return p.ParseScopeInteraction()
427 return nil, ExpectationError("expression", next)
431 // ParseScopeInteraction parses the expression types that interact
432 // with the evaluation scope: variable access, function calls, and
435 // Indexing should actually be a distinct operator in its own right,
436 // so that e.g. it can be applied to the result of a function call,
437 // but for now we're preserving the behavior of the older yacc-based
439 func (p *parser) ParseScopeInteraction() (ast.Node, error) {
440 first := p.peeker.Read()
441 startPos := first.Pos
442 if first.Type != scanner.IDENTIFIER {
443 return nil, ExpectationError("identifier", first)
446 next := p.peeker.Peek()
447 if next.Type == scanner.OPAREN {
449 funcName := first.Content
450 p.peeker.Read() // eat paren
454 if p.peeker.Peek().Type == scanner.CPAREN {
458 arg, err := p.ParseExpression()
463 args = append(args, arg)
465 if p.peeker.Peek().Type == scanner.COMMA {
466 p.peeker.Read() // eat comma
473 err := p.requireTokenType(scanner.CPAREN, `")"`)
485 varNode := &ast.VariableAccess{
490 if p.peeker.Peek().Type == scanner.OBRACKET {
492 startPos := p.peeker.Read().Pos // eat bracket
493 indexExpr, err := p.ParseExpression()
497 err = p.requireTokenType(scanner.CBRACKET, `"]"`)
511 // requireTokenType consumes the next token an returns an error if its
512 // type does not match the given type. nil is returned if the type matches.
514 // This is a helper around peeker.Read() for situations where the parser just
515 // wants to assert that a particular token type must be present.
516 func (p *parser) requireTokenType(wantType scanner.TokenType, wantName string) error {
517 token := p.peeker.Read()
518 if token.Type != wantType {
519 return ExpectationError(wantName, token)