9 "github.com/apparentlymart/go-textseg/textseg"
10 "github.com/hashicorp/hcl2/hcl"
11 "github.com/zclconf/go-cty/cty"
12 "github.com/zclconf/go-cty/cty/convert"
18 // set to true if any recovery is attempted. The parser can use this
19 // to attempt to reduce error noise by suppressing "bad token" errors
20 // in recovery mode, assuming that the recovery heuristics have failed
21 // in this case and left the peeker in a wrong place.
25 func (p *parser) ParseBody(end TokenType) (*Body, hcl.Diagnostics) {
28 var diags hcl.Diagnostics
30 startRange := p.PrevRange()
31 var endRange hcl.Range
37 endRange = p.NextRange()
47 item, itemDiags := p.ParseBodyItem()
48 diags = append(diags, itemDiags...)
49 switch titem := item.(type) {
51 blocks = append(blocks, titem)
53 if existing, exists := attrs[titem.Name]; exists {
54 diags = append(diags, &hcl.Diagnostic{
55 Severity: hcl.DiagError,
56 Summary: "Attribute redefined",
58 "The attribute %q was already defined at %s. Each attribute may be defined only once.",
59 titem.Name, existing.NameRange.String(),
61 Subject: &titem.NameRange,
64 attrs[titem.Name] = titem
67 // This should never happen for valid input, but may if a
68 // syntax error was detected in ParseBodyItem that prevented
69 // it from even producing a partially-broken item. In that
70 // case, it would've left at least one error in the diagnostics
71 // slice we already dealt with above.
73 // We'll assume ParseBodyItem attempted recovery to leave
74 // us in a reasonable position to try parsing the next item.
80 if bad.Type == TokenOQuote {
81 diags = append(diags, &hcl.Diagnostic{
82 Severity: hcl.DiagError,
83 Summary: "Invalid attribute name",
84 Detail: "Attribute names must not be quoted.",
88 diags = append(diags, &hcl.Diagnostic{
89 Severity: hcl.DiagError,
90 Summary: "Attribute or block definition required",
91 Detail: "An attribute or block definition is required here.",
96 endRange = p.PrevRange() // arbitrary, but somewhere inside the body means better diagnostics
98 p.recover(end) // attempt to recover to the token after the end of this body
107 SrcRange: hcl.RangeBetween(startRange, endRange),
109 Filename: endRange.Filename,
116 func (p *parser) ParseBodyItem() (Node, hcl.Diagnostics) {
118 if ident.Type != TokenIdent {
119 p.recoverAfterBodyItem()
120 return nil, hcl.Diagnostics{
122 Severity: hcl.DiagError,
123 Summary: "Attribute or block definition required",
124 Detail: "An attribute or block definition is required here.",
125 Subject: &ident.Range,
134 return p.finishParsingBodyAttribute(ident)
135 case TokenOQuote, TokenOBrace, TokenIdent:
136 return p.finishParsingBodyBlock(ident)
138 p.recoverAfterBodyItem()
139 return nil, hcl.Diagnostics{
141 Severity: hcl.DiagError,
142 Summary: "Attribute or block definition required",
143 Detail: "An attribute or block definition is required here. To define an attribute, use the equals sign \"=\" to introduce the attribute value.",
144 Subject: &ident.Range,
152 func (p *parser) finishParsingBodyAttribute(ident Token) (Node, hcl.Diagnostics) {
153 eqTok := p.Read() // eat equals token
154 if eqTok.Type != TokenEqual {
155 // should never happen if caller behaves
156 panic("finishParsingBodyAttribute called with next not equals")
159 var endRange hcl.Range
161 expr, diags := p.ParseExpression()
162 if p.recovery && diags.HasErrors() {
163 // recovery within expressions tends to be tricky, so we've probably
164 // landed somewhere weird. We'll try to reset to the start of a body
165 // item so parsing can continue.
166 endRange = p.PrevRange()
167 p.recoverAfterBodyItem()
170 if end.Type != TokenNewline && end.Type != TokenEOF {
172 diags = append(diags, &hcl.Diagnostic{
173 Severity: hcl.DiagError,
174 Summary: "Missing newline after attribute definition",
175 Detail: "An attribute definition must end with a newline.",
177 Context: hcl.RangeBetween(ident.Range, end.Range).Ptr(),
180 endRange = p.PrevRange()
181 p.recoverAfterBodyItem()
183 endRange = p.PrevRange()
184 p.Read() // eat newline
189 Name: string(ident.Bytes),
192 SrcRange: hcl.RangeBetween(ident.Range, endRange),
193 NameRange: ident.Range,
194 EqualsRange: eqTok.Range,
198 func (p *parser) finishParsingBodyBlock(ident Token) (Node, hcl.Diagnostics) {
199 var blockType = string(ident.Bytes)
200 var diags hcl.Diagnostics
202 var labelRanges []hcl.Range
217 label, labelRange, labelDiags := p.parseQuotedStringLiteral()
218 diags = append(diags, labelDiags...)
219 labels = append(labels, label)
220 labelRanges = append(labelRanges, labelRange)
221 if labelDiags.HasErrors() {
222 p.recoverAfterBodyItem()
228 TypeRange: ident.Range,
229 LabelRanges: labelRanges,
230 OpenBraceRange: ident.Range, // placeholder
231 CloseBraceRange: ident.Range, // placeholder
236 tok = p.Read() // eat token
237 label, labelRange := string(tok.Bytes), tok.Range
238 labels = append(labels, label)
239 labelRanges = append(labelRanges, labelRange)
244 diags = append(diags, &hcl.Diagnostic{
245 Severity: hcl.DiagError,
246 Summary: "Invalid block definition",
247 Detail: "The equals sign \"=\" indicates an attribute definition, and must not be used when defining a block.",
249 Context: hcl.RangeBetween(ident.Range, tok.Range).Ptr(),
252 diags = append(diags, &hcl.Diagnostic{
253 Severity: hcl.DiagError,
254 Summary: "Invalid block definition",
255 Detail: "A block definition must have block content delimited by \"{\" and \"}\", starting on the same line as the block header.",
257 Context: hcl.RangeBetween(ident.Range, tok.Range).Ptr(),
261 diags = append(diags, &hcl.Diagnostic{
262 Severity: hcl.DiagError,
263 Summary: "Invalid block definition",
264 Detail: "Either a quoted string block label or an opening brace (\"{\") is expected here.",
266 Context: hcl.RangeBetween(ident.Range, tok.Range).Ptr(),
271 p.recoverAfterBodyItem()
278 TypeRange: ident.Range,
279 LabelRanges: labelRanges,
280 OpenBraceRange: ident.Range, // placeholder
281 CloseBraceRange: ident.Range, // placeholder
286 // Once we fall out here, the peeker is pointed just after our opening
287 // brace, so we can begin our nested body parsing.
288 body, bodyDiags := p.ParseBody(TokenCBrace)
289 diags = append(diags, bodyDiags...)
290 cBraceRange := p.PrevRange()
293 if eol.Type == TokenNewline || eol.Type == TokenEOF {
294 p.Read() // eat newline
297 diags = append(diags, &hcl.Diagnostic{
298 Severity: hcl.DiagError,
299 Summary: "Missing newline after block definition",
300 Detail: "A block definition must end with a newline.",
302 Context: hcl.RangeBetween(ident.Range, eol.Range).Ptr(),
305 p.recoverAfterBodyItem()
313 TypeRange: ident.Range,
314 LabelRanges: labelRanges,
315 OpenBraceRange: oBrace.Range,
316 CloseBraceRange: cBraceRange,
320 func (p *parser) ParseExpression() (Expression, hcl.Diagnostics) {
321 return p.parseTernaryConditional()
324 func (p *parser) parseTernaryConditional() (Expression, hcl.Diagnostics) {
325 // The ternary conditional operator (.. ? .. : ..) behaves somewhat
326 // like a binary operator except that the "symbol" is itself
327 // an expression enclosed in two punctuation characters.
328 // The middle expression is parsed as if the ? and : symbols
329 // were parentheses. The "rhs" (the "false expression") is then
330 // treated right-associatively so it behaves similarly to the
331 // middle in terms of precedence.
333 startRange := p.NextRange()
334 var condExpr, trueExpr, falseExpr Expression
335 var diags hcl.Diagnostics
337 condExpr, condDiags := p.parseBinaryOps(binaryOps)
338 diags = append(diags, condDiags...)
339 if p.recovery && condDiags.HasErrors() {
340 return condExpr, diags
343 questionMark := p.Peek()
344 if questionMark.Type != TokenQuestion {
345 return condExpr, diags
348 p.Read() // eat question mark
350 trueExpr, trueDiags := p.ParseExpression()
351 diags = append(diags, trueDiags...)
352 if p.recovery && trueDiags.HasErrors() {
353 return condExpr, diags
357 if colon.Type != TokenColon {
358 diags = append(diags, &hcl.Diagnostic{
359 Severity: hcl.DiagError,
360 Summary: "Missing false expression in conditional",
361 Detail: "The conditional operator (...?...:...) requires a false expression, delimited by a colon.",
362 Subject: &colon.Range,
363 Context: hcl.RangeBetween(startRange, colon.Range).Ptr(),
365 return condExpr, diags
368 p.Read() // eat colon
370 falseExpr, falseDiags := p.ParseExpression()
371 diags = append(diags, falseDiags...)
372 if p.recovery && falseDiags.HasErrors() {
373 return condExpr, diags
376 return &ConditionalExpr{
378 TrueResult: trueExpr,
379 FalseResult: falseExpr,
381 SrcRange: hcl.RangeBetween(startRange, falseExpr.Range()),
385 // parseBinaryOps calls itself recursively to work through all of the
386 // operator precedence groups, and then eventually calls parseExpressionTerm
388 func (p *parser) parseBinaryOps(ops []map[TokenType]*Operation) (Expression, hcl.Diagnostics) {
390 // We've run out of operators, so now we'll just try to parse a term.
391 return p.parseExpressionWithTraversals()
397 var lhs, rhs Expression
398 var operation *Operation
399 var diags hcl.Diagnostics
401 // Parse a term that might be the first operand of a binary
402 // operation or it might just be a standalone term.
403 // We won't know until we've parsed it and can look ahead
404 // to see if there's an operator token for this level.
405 lhs, lhsDiags := p.parseBinaryOps(remaining)
406 diags = append(diags, lhsDiags...)
407 if p.recovery && lhsDiags.HasErrors() {
411 // We'll keep eating up operators until we run out, so that operators
412 // with the same precedence will combine in a left-associative manner:
413 // a+b+c => (a+b)+c, not a+(b+c)
415 // Should we later want to have right-associative operators, a way
416 // to achieve that would be to call back up to ParseExpression here
417 // instead of iteratively parsing only the remaining operators.
422 if newOp, ok = thisLevel[next.Type]; !ok {
426 // Are we extending an expression started on the previous iteration?
427 if operation != nil {
433 SrcRange: hcl.RangeBetween(lhs.Range(), rhs.Range()),
438 p.Read() // eat operator token
439 var rhsDiags hcl.Diagnostics
440 rhs, rhsDiags = p.parseBinaryOps(remaining)
441 diags = append(diags, rhsDiags...)
442 if p.recovery && rhsDiags.HasErrors() {
447 if operation == nil {
451 return &BinaryOpExpr{
456 SrcRange: hcl.RangeBetween(lhs.Range(), rhs.Range()),
460 func (p *parser) parseExpressionWithTraversals() (Expression, hcl.Diagnostics) {
461 term, diags := p.parseExpressionTerm()
470 // Attribute access or splat
474 switch attrTok.Type {
476 attrTok = p.Read() // eat token
477 name := string(attrTok.Bytes)
478 rng := hcl.RangeBetween(dot.Range, attrTok.Range)
479 step := hcl.TraverseAttr{
484 ret = makeRelativeTraversal(ret, step, rng)
487 // This is a weird form we inherited from HIL, allowing numbers
488 // to be used as attributes as a weird way of writing [n].
489 // This was never actually a first-class thing in HIL, but
490 // HIL tolerated sequences like .0. in its variable names and
491 // calling applications like Terraform exploited that to
492 // introduce indexing syntax where none existed.
493 numTok := p.Read() // eat token
496 // This syntax is ambiguous if multiple indices are used in
497 // succession, like foo.0.1.baz: that actually parses as
498 // a fractional number 0.1. Since we're only supporting this
499 // syntax for compatibility with legacy Terraform
500 // configurations, and Terraform does not tend to have lists
501 // of lists, we'll choose to reject that here with a helpful
502 // error message, rather than failing later because the index
503 // isn't a whole number.
504 if dotIdx := bytes.IndexByte(numTok.Bytes, '.'); dotIdx >= 0 {
505 first := numTok.Bytes[:dotIdx]
506 second := numTok.Bytes[dotIdx+1:]
507 diags = append(diags, &hcl.Diagnostic{
508 Severity: hcl.DiagError,
509 Summary: "Invalid legacy index syntax",
510 Detail: fmt.Sprintf("When using the legacy index syntax, chaining two indexes together is not permitted. Use the proper index syntax instead, like [%s][%s].", first, second),
511 Subject: &attrTok.Range,
513 rng := hcl.RangeBetween(dot.Range, numTok.Range)
514 step := hcl.TraverseIndex{
518 ret = makeRelativeTraversal(ret, step, rng)
522 numVal, numDiags := p.numberLitValue(numTok)
523 diags = append(diags, numDiags...)
525 rng := hcl.RangeBetween(dot.Range, numTok.Range)
526 step := hcl.TraverseIndex{
531 ret = makeRelativeTraversal(ret, step, rng)
534 // "Attribute-only" splat expression.
535 // (This is a kinda weird construct inherited from HIL, which
536 // behaves a bit like a [*] splat except that it is only able
537 // to do attribute traversals into each of its elements,
538 // whereas foo[*] can support _any_ traversal.
539 marker := p.Read() // eat star
540 trav := make(hcl.Traversal, 0, 1)
541 var firstRange, lastRange hcl.Range
542 firstRange = p.NextRange()
543 for p.Peek().Type == TokenDot {
546 if p.Peek().Type == TokenNumberLit {
547 // Continuing the "weird stuff inherited from HIL"
548 // theme, we also allow numbers as attribute names
549 // inside splats and interpret them as indexing
550 // into a list, for expressions like:
551 // foo.bar.*.baz.0.foo
554 // Weird special case if the user writes something
555 // like foo.bar.*.baz.0.0.foo, where 0.0 parses
557 if dotIdx := bytes.IndexByte(numTok.Bytes, '.'); dotIdx >= 0 {
558 first := numTok.Bytes[:dotIdx]
559 second := numTok.Bytes[dotIdx+1:]
560 diags = append(diags, &hcl.Diagnostic{
561 Severity: hcl.DiagError,
562 Summary: "Invalid legacy index syntax",
563 Detail: fmt.Sprintf("When using the legacy index syntax, chaining two indexes together is not permitted. Use the proper index syntax with a full splat expression [*] instead, like [%s][%s].", first, second),
564 Subject: &attrTok.Range,
566 trav = append(trav, hcl.TraverseIndex{
568 SrcRange: hcl.RangeBetween(dot.Range, numTok.Range),
570 lastRange = numTok.Range
574 numVal, numDiags := p.numberLitValue(numTok)
575 diags = append(diags, numDiags...)
576 trav = append(trav, hcl.TraverseIndex{
578 SrcRange: hcl.RangeBetween(dot.Range, numTok.Range),
580 lastRange = numTok.Range
584 if p.Peek().Type != TokenIdent {
586 if p.Peek().Type == TokenStar {
587 diags = append(diags, &hcl.Diagnostic{
588 Severity: hcl.DiagError,
589 Summary: "Nested splat expression not allowed",
590 Detail: "A splat expression (*) cannot be used inside another attribute-only splat expression.",
591 Subject: p.Peek().Range.Ptr(),
594 diags = append(diags, &hcl.Diagnostic{
595 Severity: hcl.DiagError,
596 Summary: "Invalid attribute name",
597 Detail: "An attribute name is required after a dot.",
598 Subject: &attrTok.Range,
607 trav = append(trav, hcl.TraverseAttr{
608 Name: string(attrTok.Bytes),
609 SrcRange: hcl.RangeBetween(dot.Range, attrTok.Range),
611 lastRange = attrTok.Range
614 itemExpr := &AnonSymbolExpr{
615 SrcRange: hcl.RangeBetween(dot.Range, marker.Range),
617 var travExpr Expression
621 travExpr = &RelativeTraversalExpr{
624 SrcRange: hcl.RangeBetween(firstRange, lastRange),
633 SrcRange: hcl.RangeBetween(dot.Range, lastRange),
634 MarkerRange: hcl.RangeBetween(dot.Range, marker.Range),
638 diags = append(diags, &hcl.Diagnostic{
639 Severity: hcl.DiagError,
640 Summary: "Invalid attribute name",
641 Detail: "An attribute name is required after a dot.",
642 Subject: &attrTok.Range,
644 // This leaves the peeker in a bad place, so following items
645 // will probably be misparsed until we hit something that
646 // allows us to re-sync.
648 // We will probably need to do something better here eventually
649 // in order to support autocomplete triggered by typing a
655 // Indexing of a collection.
656 // This may or may not be a hcl.Traverser, depending on whether
657 // the key value is something constant.
660 // TODO: If we have a TokenStar inside our brackets, parse as
661 // a Splat expression: foo[*].baz[0].
663 p.PushIncludeNewlines(false) // arbitrary newlines allowed in brackets
664 keyExpr, keyDiags := p.ParseExpression()
665 diags = append(diags, keyDiags...)
666 if p.recovery && keyDiags.HasErrors() {
667 close = p.recover(TokenCBrack)
670 if close.Type != TokenCBrack && !p.recovery {
671 diags = append(diags, &hcl.Diagnostic{
672 Severity: hcl.DiagError,
673 Summary: "Missing close bracket on index",
674 Detail: "The index operator must end with a closing bracket (\"]\").",
675 Subject: &close.Range,
677 close = p.recover(TokenCBrack)
680 p.PopIncludeNewlines()
682 if lit, isLit := keyExpr.(*LiteralValueExpr); isLit {
683 litKey, _ := lit.Value(nil)
684 rng := hcl.RangeBetween(open.Range, close.Range)
685 step := hcl.TraverseIndex{
689 ret = makeRelativeTraversal(ret, step, rng)
691 rng := hcl.RangeBetween(open.Range, close.Range)
697 OpenRange: open.Range,
709 // makeRelativeTraversal takes an expression and a traverser and returns
710 // a traversal expression that combines the two. If the given expression
711 // is already a traversal, it is extended in place (mutating it) and
712 // returned. If it isn't, a new RelativeTraversalExpr is created and returned.
713 func makeRelativeTraversal(expr Expression, next hcl.Traverser, rng hcl.Range) Expression {
714 switch texpr := expr.(type) {
715 case *ScopeTraversalExpr:
716 texpr.Traversal = append(texpr.Traversal, next)
717 texpr.SrcRange = hcl.RangeBetween(texpr.SrcRange, rng)
719 case *RelativeTraversalExpr:
720 texpr.Traversal = append(texpr.Traversal, next)
721 texpr.SrcRange = hcl.RangeBetween(texpr.SrcRange, rng)
724 return &RelativeTraversalExpr{
726 Traversal: hcl.Traversal{next},
732 func (p *parser) parseExpressionTerm() (Expression, hcl.Diagnostics) {
737 p.Read() // eat open paren
739 p.PushIncludeNewlines(false)
741 expr, diags := p.ParseExpression()
742 if diags.HasErrors() {
743 // attempt to place the peeker after our closing paren
744 // before we return, so that the next parser has some
745 // chance of finding a valid expression.
746 p.recover(TokenCParen)
747 p.PopIncludeNewlines()
752 if close.Type != TokenCParen {
753 diags = append(diags, &hcl.Diagnostic{
754 Severity: hcl.DiagError,
755 Summary: "Unbalanced parentheses",
756 Detail: "Expected a closing parenthesis to terminate the expression.",
757 Subject: &close.Range,
758 Context: hcl.RangeBetween(start.Range, close.Range).Ptr(),
763 p.Read() // eat closing paren
764 p.PopIncludeNewlines()
769 tok := p.Read() // eat number token
771 numVal, diags := p.numberLitValue(tok)
772 return &LiteralValueExpr{
778 tok := p.Read() // eat identifier token
780 if p.Peek().Type == TokenOParen {
781 return p.finishParsingFunctionCall(tok)
784 name := string(tok.Bytes)
787 return &LiteralValueExpr{
792 return &LiteralValueExpr{
797 return &LiteralValueExpr{
798 Val: cty.NullVal(cty.DynamicPseudoType),
802 return &ScopeTraversalExpr{
803 Traversal: hcl.Traversal{
813 case TokenOQuote, TokenOHeredoc:
814 open := p.Read() // eat opening marker
815 closer := p.oppositeBracket(open.Type)
816 exprs, passthru, _, diags := p.parseTemplateInner(closer)
818 closeRange := p.PrevRange()
822 panic("passthru set with len(exprs) != 1")
824 return &TemplateWrapExpr{
826 SrcRange: hcl.RangeBetween(open.Range, closeRange),
830 return &TemplateExpr{
832 SrcRange: hcl.RangeBetween(open.Range, closeRange),
836 tok := p.Read() // eat minus token
838 // Important to use parseExpressionWithTraversals rather than parseExpression
839 // here, otherwise we can capture a following binary expression into
841 // e.g. -46+5 should parse as (-46)+5, not -(46+5)
842 operand, diags := p.parseExpressionWithTraversals()
847 SrcRange: hcl.RangeBetween(tok.Range, operand.Range()),
848 SymbolRange: tok.Range,
852 tok := p.Read() // eat bang token
854 // Important to use parseExpressionWithTraversals rather than parseExpression
855 // here, otherwise we can capture a following binary expression into
857 operand, diags := p.parseExpressionWithTraversals()
862 SrcRange: hcl.RangeBetween(tok.Range, operand.Range()),
863 SymbolRange: tok.Range,
867 return p.parseTupleCons()
870 return p.parseObjectCons()
873 var diags hcl.Diagnostics
875 diags = append(diags, &hcl.Diagnostic{
876 Severity: hcl.DiagError,
877 Summary: "Invalid expression",
878 Detail: "Expected the start of an expression, but found an invalid expression token.",
879 Subject: &start.Range,
884 // Return a placeholder so that the AST is still structurally sound
885 // even in the presence of parse errors.
886 return &LiteralValueExpr{
888 SrcRange: start.Range,
893 func (p *parser) numberLitValue(tok Token) (cty.Value, hcl.Diagnostics) {
894 // We'll lean on the cty converter to do the conversion, to ensure that
895 // the behavior is the same as what would happen if converting a
896 // non-literal string to a number.
897 numStrVal := cty.StringVal(string(tok.Bytes))
898 numVal, err := convert.Convert(numStrVal, cty.Number)
900 ret := cty.UnknownVal(cty.Number)
901 return ret, hcl.Diagnostics{
903 Severity: hcl.DiagError,
904 Summary: "Invalid number literal",
905 // FIXME: not a very good error message, but convert only
906 // gives us "a number is required", so not much help either.
907 Detail: "Failed to recognize the value of this number literal.",
915 // finishParsingFunctionCall parses a function call assuming that the function
916 // name was already read, and so the peeker should be pointing at the opening
917 // parenthesis after the name.
918 func (p *parser) finishParsingFunctionCall(name Token) (Expression, hcl.Diagnostics) {
920 if openTok.Type != TokenOParen {
921 // should never happen if callers behave
922 panic("finishParsingFunctionCall called with non-parenthesis as next token")
925 var args []Expression
926 var diags hcl.Diagnostics
930 // Arbitrary newlines are allowed inside the function call parentheses.
931 p.PushIncludeNewlines(false)
937 if tok.Type == TokenCParen {
938 closeTok = p.Read() // eat closing paren
942 arg, argDiags := p.ParseExpression()
943 args = append(args, arg)
944 diags = append(diags, argDiags...)
945 if p.recovery && argDiags.HasErrors() {
946 // if there was a parse error in the argument then we've
947 // probably been left in a weird place in the token stream,
948 // so we'll bail out with a partial argument list.
949 p.recover(TokenCParen)
954 if sep.Type == TokenCParen {
959 if sep.Type == TokenEllipsis {
962 if p.Peek().Type != TokenCParen {
964 diags = append(diags, &hcl.Diagnostic{
965 Severity: hcl.DiagError,
966 Summary: "Missing closing parenthesis",
967 Detail: "An expanded function argument (with ...) must be immediately followed by closing parentheses.",
969 Context: hcl.RangeBetween(name.Range, sep.Range).Ptr(),
972 closeTok = p.recover(TokenCParen)
974 closeTok = p.Read() // eat closing paren
979 if sep.Type != TokenComma {
980 diags = append(diags, &hcl.Diagnostic{
981 Severity: hcl.DiagError,
982 Summary: "Missing argument separator",
983 Detail: "A comma is required to separate each function argument from the next.",
985 Context: hcl.RangeBetween(name.Range, sep.Range).Ptr(),
987 closeTok = p.recover(TokenCParen)
991 if p.Peek().Type == TokenCParen {
992 // A trailing comma after the last argument gets us in here.
993 closeTok = p.Read() // eat closing paren
999 p.PopIncludeNewlines()
1001 return &FunctionCallExpr{
1002 Name: string(name.Bytes),
1005 ExpandFinal: expandFinal,
1007 NameRange: name.Range,
1008 OpenParenRange: openTok.Range,
1009 CloseParenRange: closeTok.Range,
1013 func (p *parser) parseTupleCons() (Expression, hcl.Diagnostics) {
1015 if open.Type != TokenOBrack {
1016 // Should never happen if callers are behaving
1017 panic("parseTupleCons called without peeker pointing to open bracket")
1020 p.PushIncludeNewlines(false)
1021 defer p.PopIncludeNewlines()
1023 if forKeyword.TokenMatches(p.Peek()) {
1024 return p.finishParsingForExpr(open)
1029 var diags hcl.Diagnostics
1030 var exprs []Expression
1034 if next.Type == TokenCBrack {
1035 close = p.Read() // eat closer
1039 expr, exprDiags := p.ParseExpression()
1040 exprs = append(exprs, expr)
1041 diags = append(diags, exprDiags...)
1043 if p.recovery && exprDiags.HasErrors() {
1044 // If expression parsing failed then we are probably in a strange
1045 // place in the token stream, so we'll bail out and try to reset
1046 // to after our closing bracket to allow parsing to continue.
1047 close = p.recover(TokenCBrack)
1052 if next.Type == TokenCBrack {
1053 close = p.Read() // eat closer
1057 if next.Type != TokenComma {
1059 diags = append(diags, &hcl.Diagnostic{
1060 Severity: hcl.DiagError,
1061 Summary: "Missing item separator",
1062 Detail: "Expected a comma to mark the beginning of the next item.",
1063 Subject: &next.Range,
1064 Context: hcl.RangeBetween(open.Range, next.Range).Ptr(),
1067 close = p.recover(TokenCBrack)
1071 p.Read() // eat comma
1075 return &TupleConsExpr{
1078 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1079 OpenRange: open.Range,
1083 func (p *parser) parseObjectCons() (Expression, hcl.Diagnostics) {
1085 if open.Type != TokenOBrace {
1086 // Should never happen if callers are behaving
1087 panic("parseObjectCons called without peeker pointing to open brace")
1090 p.PushIncludeNewlines(true)
1091 defer p.PopIncludeNewlines()
1093 if forKeyword.TokenMatches(p.Peek()) {
1094 return p.finishParsingForExpr(open)
1099 var diags hcl.Diagnostics
1100 var items []ObjectConsItem
1104 if next.Type == TokenNewline {
1105 p.Read() // eat newline
1109 if next.Type == TokenCBrace {
1110 close = p.Read() // eat closer
1115 var keyDiags hcl.Diagnostics
1116 key, keyDiags = p.ParseExpression()
1117 diags = append(diags, keyDiags...)
1119 if p.recovery && keyDiags.HasErrors() {
1120 // If expression parsing failed then we are probably in a strange
1121 // place in the token stream, so we'll bail out and try to reset
1122 // to after our closing brace to allow parsing to continue.
1123 close = p.recover(TokenCBrace)
1127 // We wrap up the key expression in a special wrapper that deals
1128 // with our special case that naked identifiers as object keys
1129 // are interpreted as literal strings.
1130 key = &ObjectConsKeyExpr{Wrapped: key}
1133 if next.Type != TokenEqual && next.Type != TokenColon {
1135 if next.Type == TokenNewline || next.Type == TokenComma {
1136 diags = append(diags, &hcl.Diagnostic{
1137 Severity: hcl.DiagError,
1138 Summary: "Missing item value",
1139 Detail: "Expected an item value, introduced by an equals sign (\"=\").",
1140 Subject: &next.Range,
1141 Context: hcl.RangeBetween(open.Range, next.Range).Ptr(),
1144 diags = append(diags, &hcl.Diagnostic{
1145 Severity: hcl.DiagError,
1146 Summary: "Missing key/value separator",
1147 Detail: "Expected an equals sign (\"=\") to mark the beginning of the item value.",
1148 Subject: &next.Range,
1149 Context: hcl.RangeBetween(open.Range, next.Range).Ptr(),
1153 close = p.recover(TokenCBrace)
1157 p.Read() // eat equals sign or colon
1159 value, valueDiags := p.ParseExpression()
1160 diags = append(diags, valueDiags...)
1162 if p.recovery && valueDiags.HasErrors() {
1163 // If expression parsing failed then we are probably in a strange
1164 // place in the token stream, so we'll bail out and try to reset
1165 // to after our closing brace to allow parsing to continue.
1166 close = p.recover(TokenCBrace)
1170 items = append(items, ObjectConsItem{
1176 if next.Type == TokenCBrace {
1177 close = p.Read() // eat closer
1181 if next.Type != TokenComma && next.Type != TokenNewline {
1183 diags = append(diags, &hcl.Diagnostic{
1184 Severity: hcl.DiagError,
1185 Summary: "Missing item separator",
1186 Detail: "Expected a newline or comma to mark the beginning of the next item.",
1187 Subject: &next.Range,
1188 Context: hcl.RangeBetween(open.Range, next.Range).Ptr(),
1191 close = p.recover(TokenCBrace)
1195 p.Read() // eat comma or newline
1199 return &ObjectConsExpr{
1202 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1203 OpenRange: open.Range,
1207 func (p *parser) finishParsingForExpr(open Token) (Expression, hcl.Diagnostics) {
1208 introducer := p.Read()
1209 if !forKeyword.TokenMatches(introducer) {
1210 // Should never happen if callers are behaving
1211 panic("finishParsingForExpr called without peeker pointing to 'for' identifier")
1215 var closeType TokenType
1219 closeType = TokenCBrace
1221 makeObj = false // making a tuple
1222 closeType = TokenCBrack
1224 // Should never happen if callers are behaving
1225 panic("finishParsingForExpr called with invalid open token")
1228 var diags hcl.Diagnostics
1229 var keyName, valName string
1231 if p.Peek().Type != TokenIdent {
1233 diags = append(diags, &hcl.Diagnostic{
1234 Severity: hcl.DiagError,
1235 Summary: "Invalid 'for' expression",
1236 Detail: "For expression requires variable name after 'for'.",
1237 Subject: p.Peek().Range.Ptr(),
1238 Context: hcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
1241 close := p.recover(closeType)
1242 return &LiteralValueExpr{
1243 Val: cty.DynamicVal,
1244 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1248 valName = string(p.Read().Bytes)
1250 if p.Peek().Type == TokenComma {
1251 // What we just read was actually the key, then.
1253 p.Read() // eat comma
1255 if p.Peek().Type != TokenIdent {
1257 diags = append(diags, &hcl.Diagnostic{
1258 Severity: hcl.DiagError,
1259 Summary: "Invalid 'for' expression",
1260 Detail: "For expression requires value variable name after comma.",
1261 Subject: p.Peek().Range.Ptr(),
1262 Context: hcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
1265 close := p.recover(closeType)
1266 return &LiteralValueExpr{
1267 Val: cty.DynamicVal,
1268 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1272 valName = string(p.Read().Bytes)
1275 if !inKeyword.TokenMatches(p.Peek()) {
1277 diags = append(diags, &hcl.Diagnostic{
1278 Severity: hcl.DiagError,
1279 Summary: "Invalid 'for' expression",
1280 Detail: "For expression requires 'in' keyword after names.",
1281 Subject: p.Peek().Range.Ptr(),
1282 Context: hcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
1285 close := p.recover(closeType)
1286 return &LiteralValueExpr{
1287 Val: cty.DynamicVal,
1288 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1291 p.Read() // eat 'in' keyword
1293 collExpr, collDiags := p.ParseExpression()
1294 diags = append(diags, collDiags...)
1295 if p.recovery && collDiags.HasErrors() {
1296 close := p.recover(closeType)
1297 return &LiteralValueExpr{
1298 Val: cty.DynamicVal,
1299 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1303 if p.Peek().Type != TokenColon {
1305 diags = append(diags, &hcl.Diagnostic{
1306 Severity: hcl.DiagError,
1307 Summary: "Invalid 'for' expression",
1308 Detail: "For expression requires colon after collection expression.",
1309 Subject: p.Peek().Range.Ptr(),
1310 Context: hcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
1313 close := p.recover(closeType)
1314 return &LiteralValueExpr{
1315 Val: cty.DynamicVal,
1316 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1319 p.Read() // eat colon
1321 var keyExpr, valExpr Expression
1322 var keyDiags, valDiags hcl.Diagnostics
1323 valExpr, valDiags = p.ParseExpression()
1324 if p.Peek().Type == TokenFatArrow {
1325 // What we just parsed was actually keyExpr
1326 p.Read() // eat the fat arrow
1327 keyExpr, keyDiags = valExpr, valDiags
1329 valExpr, valDiags = p.ParseExpression()
1331 diags = append(diags, keyDiags...)
1332 diags = append(diags, valDiags...)
1333 if p.recovery && (keyDiags.HasErrors() || valDiags.HasErrors()) {
1334 close := p.recover(closeType)
1335 return &LiteralValueExpr{
1336 Val: cty.DynamicVal,
1337 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1343 if p.Peek().Type == TokenEllipsis {
1348 var condExpr Expression
1349 var condDiags hcl.Diagnostics
1350 if ifKeyword.TokenMatches(p.Peek()) {
1351 p.Read() // eat "if"
1352 condExpr, condDiags = p.ParseExpression()
1353 diags = append(diags, condDiags...)
1354 if p.recovery && condDiags.HasErrors() {
1355 close := p.recover(p.oppositeBracket(open.Type))
1356 return &LiteralValueExpr{
1357 Val: cty.DynamicVal,
1358 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1364 if p.Peek().Type == closeType {
1368 diags = append(diags, &hcl.Diagnostic{
1369 Severity: hcl.DiagError,
1370 Summary: "Invalid 'for' expression",
1371 Detail: "Extra characters after the end of the 'for' expression.",
1372 Subject: p.Peek().Range.Ptr(),
1373 Context: hcl.RangeBetween(open.Range, p.Peek().Range).Ptr(),
1376 close = p.recover(closeType)
1381 diags = append(diags, &hcl.Diagnostic{
1382 Severity: hcl.DiagError,
1383 Summary: "Invalid 'for' expression",
1384 Detail: "Key expression is not valid when building a tuple.",
1385 Subject: keyExpr.Range().Ptr(),
1386 Context: hcl.RangeBetween(open.Range, close.Range).Ptr(),
1391 diags = append(diags, &hcl.Diagnostic{
1392 Severity: hcl.DiagError,
1393 Summary: "Invalid 'for' expression",
1394 Detail: "Grouping ellipsis (...) cannot be used when building a tuple.",
1395 Subject: &ellipsis.Range,
1396 Context: hcl.RangeBetween(open.Range, close.Range).Ptr(),
1401 diags = append(diags, &hcl.Diagnostic{
1402 Severity: hcl.DiagError,
1403 Summary: "Invalid 'for' expression",
1404 Detail: "Key expression is required when building an object.",
1405 Subject: valExpr.Range().Ptr(),
1406 Context: hcl.RangeBetween(open.Range, close.Range).Ptr(),
1420 SrcRange: hcl.RangeBetween(open.Range, close.Range),
1421 OpenRange: open.Range,
1422 CloseRange: close.Range,
1426 // parseQuotedStringLiteral is a helper for parsing quoted strings that
1427 // aren't allowed to contain any interpolations, such as block labels.
1428 func (p *parser) parseQuotedStringLiteral() (string, hcl.Range, hcl.Diagnostics) {
1430 if oQuote.Type != TokenOQuote {
1431 return "", oQuote.Range, hcl.Diagnostics{
1433 Severity: hcl.DiagError,
1434 Summary: "Invalid string literal",
1435 Detail: "A quoted string is required here.",
1436 Subject: &oQuote.Range,
1441 var diags hcl.Diagnostics
1442 ret := &bytes.Buffer{}
1454 case TokenQuotedLit:
1455 s, sDiags := p.decodeStringLit(tok)
1456 diags = append(diags, sDiags...)
1459 case TokenTemplateControl, TokenTemplateInterp:
1461 if tok.Type == TokenTemplateControl {
1465 diags = append(diags, &hcl.Diagnostic{
1466 Severity: hcl.DiagError,
1467 Summary: "Invalid string literal",
1468 Detail: fmt.Sprintf(
1469 "Template sequences are not allowed in this string. To include a literal %q, double it (as \"%s%s\") to escape it.",
1470 which, which, which,
1472 Subject: &tok.Range,
1473 Context: hcl.RangeBetween(oQuote.Range, tok.Range).Ptr(),
1475 p.recover(TokenTemplateSeqEnd)
1478 diags = append(diags, &hcl.Diagnostic{
1479 Severity: hcl.DiagError,
1480 Summary: "Unterminated string literal",
1481 Detail: "Unable to find the closing quote mark before the end of the file.",
1482 Subject: &tok.Range,
1483 Context: hcl.RangeBetween(oQuote.Range, tok.Range).Ptr(),
1488 // Should never happen, as long as the scanner is behaving itself
1489 diags = append(diags, &hcl.Diagnostic{
1490 Severity: hcl.DiagError,
1491 Summary: "Invalid string literal",
1492 Detail: "This item is not valid in a string literal.",
1493 Subject: &tok.Range,
1494 Context: hcl.RangeBetween(oQuote.Range, tok.Range).Ptr(),
1496 p.recover(TokenOQuote)
1503 return ret.String(), hcl.RangeBetween(oQuote.Range, cQuote.Range), diags
1506 // decodeStringLit processes the given token, which must be either a
1507 // TokenQuotedLit or a TokenStringLit, returning the string resulting from
1508 // resolving any escape sequences.
1510 // If any error diagnostics are returned, the returned string may be incomplete
1511 // or otherwise invalid.
1512 func (p *parser) decodeStringLit(tok Token) (string, hcl.Diagnostics) {
1515 case TokenQuotedLit:
1517 case TokenStringLit:
1520 panic("decodeQuotedLit can only be used with TokenStringLit and TokenQuotedLit tokens")
1522 var diags hcl.Diagnostics
1524 ret := make([]byte, 0, len(tok.Bytes))
1525 slices := scanStringLit(tok.Bytes, quoted)
1527 // We will mutate rng constantly as we walk through our token slices below.
1528 // Any diagnostics must take a copy of this rng rather than simply pointing
1529 // to it, e.g. by using rng.Ptr() rather than &rng.
1534 for _, slice := range slices {
1535 if len(slice) == 0 {
1539 // Advance the start of our range to where the previous token ended
1542 // Advance the end of our range to after our token.
1545 adv, ch, _ := textseg.ScanGraphemeClusters(b, true)
1561 // If we're not in quoted mode then just treat this token as
1562 // normal. (Slices can still start with backslash even if we're
1563 // not specifically looking for backslash sequences.)
1567 diags = append(diags, &hcl.Diagnostic{
1568 Severity: hcl.DiagError,
1569 Summary: "Invalid escape sequence",
1570 Detail: "Backslash must be followed by an escape sequence selector character.",
1579 ret = append(ret, '\n')
1582 ret = append(ret, '\r')
1585 ret = append(ret, '\t')
1588 ret = append(ret, '"')
1591 ret = append(ret, '\\')
1594 if slice[1] == 'u' && len(slice) != 6 {
1595 diags = append(diags, &hcl.Diagnostic{
1596 Severity: hcl.DiagError,
1597 Summary: "Invalid escape sequence",
1598 Detail: "The \\u escape sequence must be followed by four hexadecimal digits.",
1602 } else if slice[1] == 'U' && len(slice) != 10 {
1603 diags = append(diags, &hcl.Diagnostic{
1604 Severity: hcl.DiagError,
1605 Summary: "Invalid escape sequence",
1606 Detail: "The \\U escape sequence must be followed by eight hexadecimal digits.",
1612 numHex := string(slice[2:])
1613 num, err := strconv.ParseUint(numHex, 16, 32)
1615 // Should never happen because the scanner won't match
1616 // a sequence of digits that isn't valid.
1621 l := utf8.RuneLen(r)
1623 diags = append(diags, &hcl.Diagnostic{
1624 Severity: hcl.DiagError,
1625 Summary: "Invalid escape sequence",
1626 Detail: fmt.Sprintf("Cannot encode character U+%04x in UTF-8.", num),
1631 for i := 0; i < l; i++ {
1632 ret = append(ret, 0)
1634 rb := ret[len(ret)-l:]
1635 utf8.EncodeRune(rb, r)
1640 diags = append(diags, &hcl.Diagnostic{
1641 Severity: hcl.DiagError,
1642 Summary: "Invalid escape sequence",
1643 Detail: fmt.Sprintf("The symbol %q is not a valid escape sequence selector.", slice[1:]),
1646 ret = append(ret, slice[1:]...)
1651 if len(slice) != 3 {
1652 // Not long enough to be our escape sequence, so it's literal.
1656 if slice[1] == slice[0] && slice[2] == '{' {
1657 ret = append(ret, slice[0])
1658 ret = append(ret, '{')
1665 // If we fall out here or break out of here from the switch above
1666 // then this slice is just a literal.
1667 ret = append(ret, slice...)
1670 return string(ret), diags
1673 // setRecovery turns on recovery mode without actually doing any recovery.
1674 // This can be used when a parser knowingly leaves the peeker in a useless
1675 // place and wants to suppress errors that might result from that decision.
1676 func (p *parser) setRecovery() {
1680 // recover seeks forward in the token stream until it finds TokenType "end",
1681 // then returns with the peeker pointed at the following token.
1683 // If the given token type is a bracketer, this function will additionally
1684 // count nested instances of the brackets to try to leave the peeker at
1685 // the end of the _current_ instance of that bracketer, skipping over any
1686 // nested instances. This is a best-effort operation and may have
1687 // unpredictable results on input with bad bracketer nesting.
1688 func (p *parser) recover(end TokenType) Token {
1689 start := p.oppositeBracket(end)
1696 if end == TokenTemplateSeqEnd && ty == TokenTemplateControl {
1697 // normalize so that our matching behavior can work, since
1698 // TokenTemplateControl/TokenTemplateInterp are asymmetrical
1699 // with TokenTemplateSeqEnd and thus we need to count both
1700 // openers if that's the closer we're looking for.
1701 ty = TokenTemplateInterp
1719 // recoverOver seeks forward in the token stream until it finds a block
1720 // starting with TokenType "start", then finds the corresponding end token,
1721 // leaving the peeker pointed at the token after that end token.
1723 // The given token type _must_ be a bracketer. For example, if the given
1724 // start token is TokenOBrace then the parser will be left at the _end_ of
1725 // the next brace-delimited block encountered, or at EOF if no such block
1726 // is found or it is unclosed.
1727 func (p *parser) recoverOver(start TokenType) {
1728 end := p.oppositeBracket(start)
1730 // find the opening bracket first
1735 case start, TokenEOF:
1740 // Now use our existing recover function to locate the _end_ of the
1741 // container we've found.
1745 func (p *parser) recoverAfterBodyItem() {
1747 var open []TokenType
1763 case TokenOBrace, TokenOBrack, TokenOParen, TokenOQuote, TokenOHeredoc, TokenTemplateInterp, TokenTemplateControl:
1764 open = append(open, tok.Type)
1766 case TokenCBrace, TokenCBrack, TokenCParen, TokenCQuote, TokenCHeredoc:
1767 opener := p.oppositeBracket(tok.Type)
1768 for len(open) > 0 && open[len(open)-1] != opener {
1769 open = open[:len(open)-1]
1772 open = open[:len(open)-1]
1775 case TokenTemplateSeqEnd:
1776 for len(open) > 0 && open[len(open)-1] != TokenTemplateInterp && open[len(open)-1] != TokenTemplateControl {
1777 open = open[:len(open)-1]
1780 open = open[:len(open)-1]
1787 // oppositeBracket finds the bracket that opposes the given bracketer, or
1788 // NilToken if the given token isn't a bracketer.
1790 // "Bracketer", for the sake of this function, is one end of a matching
1791 // open/close set of tokens that establish a bracketing context.
1792 func (p *parser) oppositeBracket(ty TokenType) TokenType {
1804 return TokenCHeredoc
1815 return TokenOHeredoc
1817 case TokenTemplateControl:
1818 return TokenTemplateSeqEnd
1819 case TokenTemplateInterp:
1820 return TokenTemplateSeqEnd
1821 case TokenTemplateSeqEnd:
1822 // This is ambigous, but we return Interp here because that's
1823 // what's assumed by the "recover" method.
1824 return TokenTemplateInterp
1831 func errPlaceholderExpr(rng hcl.Range) Expression {
1832 return &LiteralValueExpr{
1833 Val: cty.DynamicVal,