aboutsummaryrefslogtreecommitdiffhomepage
path: root/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go')
-rw-r--r--vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go92
1 files changed, 70 insertions, 22 deletions
diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go
index bcaa15f..3d898fd 100644
--- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go
+++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go
@@ -1,6 +1,7 @@
1package hclsyntax 1package hclsyntax
2 2
3import ( 3import (
4 "bytes"
4 "fmt" 5 "fmt"
5 6
6 "github.com/apparentlymart/go-textseg/textseg" 7 "github.com/apparentlymart/go-textseg/textseg"
@@ -84,16 +85,18 @@ const (
84 // things that might work in other languages they are familiar with, or 85 // things that might work in other languages they are familiar with, or
85 // simply make incorrect assumptions about the HCL language. 86 // simply make incorrect assumptions about the HCL language.
86 87
87 TokenBitwiseAnd TokenType = '&' 88 TokenBitwiseAnd TokenType = '&'
88 TokenBitwiseOr TokenType = '|' 89 TokenBitwiseOr TokenType = '|'
89 TokenBitwiseNot TokenType = '~' 90 TokenBitwiseNot TokenType = '~'
90 TokenBitwiseXor TokenType = '^' 91 TokenBitwiseXor TokenType = '^'
91 TokenStarStar TokenType = '➚' 92 TokenStarStar TokenType = '➚'
92 TokenBacktick TokenType = '`' 93 TokenApostrophe TokenType = '\''
93 TokenSemicolon TokenType = ';' 94 TokenBacktick TokenType = '`'
94 TokenTabs TokenType = '␉' 95 TokenSemicolon TokenType = ';'
95 TokenInvalid TokenType = '�' 96 TokenTabs TokenType = '␉'
96 TokenBadUTF8 TokenType = '💩' 97 TokenInvalid TokenType = '�'
98 TokenBadUTF8 TokenType = '💩'
99 TokenQuotedNewline TokenType = '␤'
97 100
98 // TokenNil is a placeholder for when a token is required but none is 101 // TokenNil is a placeholder for when a token is required but none is
99 // available, e.g. when reporting errors. The scanner will never produce 102 // available, e.g. when reporting errors. The scanner will never produce
@@ -114,10 +117,11 @@ const (
114) 117)
115 118
116type tokenAccum struct { 119type tokenAccum struct {
117 Filename string 120 Filename string
118 Bytes []byte 121 Bytes []byte
119 Pos hcl.Pos 122 Pos hcl.Pos
120 Tokens []Token 123 Tokens []Token
124 StartByte int
121} 125}
122 126
123func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) { 127func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) {
@@ -125,11 +129,11 @@ func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) {
125 // the start pos to get our end pos. 129 // the start pos to get our end pos.
126 130
127 start := f.Pos 131 start := f.Pos
128 start.Column += startOfs - f.Pos.Byte // Safe because only ASCII spaces can be in the offset 132 start.Column += startOfs + f.StartByte - f.Pos.Byte // Safe because only ASCII spaces can be in the offset
129 start.Byte = startOfs 133 start.Byte = startOfs + f.StartByte
130 134
131 end := start 135 end := start
132 end.Byte = endOfs 136 end.Byte = endOfs + f.StartByte
133 b := f.Bytes[startOfs:endOfs] 137 b := f.Bytes[startOfs:endOfs]
134 for len(b) > 0 { 138 for len(b) > 0 {
135 advance, seq, _ := textseg.ScanGraphemeClusters(b, true) 139 advance, seq, _ := textseg.ScanGraphemeClusters(b, true)
@@ -160,6 +164,13 @@ type heredocInProgress struct {
160 StartOfLine bool 164 StartOfLine bool
161} 165}
162 166
167func tokenOpensFlushHeredoc(tok Token) bool {
168 if tok.Type != TokenOHeredoc {
169 return false
170 }
171 return bytes.HasPrefix(tok.Bytes, []byte{'<', '<', '-'})
172}
173
163// checkInvalidTokens does a simple pass across the given tokens and generates 174// checkInvalidTokens does a simple pass across the given tokens and generates
164// diagnostics for tokens that should _never_ appear in HCL source. This 175// diagnostics for tokens that should _never_ appear in HCL source. This
165// is intended to avoid the need for the parser to have special support 176// is intended to avoid the need for the parser to have special support
@@ -174,11 +185,15 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics {
174 toldBitwise := 0 185 toldBitwise := 0
175 toldExponent := 0 186 toldExponent := 0
176 toldBacktick := 0 187 toldBacktick := 0
188 toldApostrophe := 0
177 toldSemicolon := 0 189 toldSemicolon := 0
178 toldTabs := 0 190 toldTabs := 0
179 toldBadUTF8 := 0 191 toldBadUTF8 := 0
180 192
181 for _, tok := range tokens { 193 for _, tok := range tokens {
194 // copy token so it's safe to point to it
195 tok := tok
196
182 switch tok.Type { 197 switch tok.Type {
183 case TokenBitwiseAnd, TokenBitwiseOr, TokenBitwiseXor, TokenBitwiseNot: 198 case TokenBitwiseAnd, TokenBitwiseOr, TokenBitwiseXor, TokenBitwiseNot:
184 if toldBitwise < 4 { 199 if toldBitwise < 4 {
@@ -214,22 +229,36 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics {
214 case TokenBacktick: 229 case TokenBacktick:
215 // Only report for alternating (even) backticks, so we won't report both start and ends of the same 230 // Only report for alternating (even) backticks, so we won't report both start and ends of the same
216 // backtick-quoted string. 231 // backtick-quoted string.
217 if toldExponent < 4 && (toldExponent%2) == 0 { 232 if (toldBacktick % 2) == 0 {
218 diags = append(diags, &hcl.Diagnostic{ 233 diags = append(diags, &hcl.Diagnostic{
219 Severity: hcl.DiagError, 234 Severity: hcl.DiagError,
220 Summary: "Invalid character", 235 Summary: "Invalid character",
221 Detail: "The \"`\" character is not valid. To create a multi-line string, use the \"heredoc\" syntax, like \"<<EOT\".", 236 Detail: "The \"`\" character is not valid. To create a multi-line string, use the \"heredoc\" syntax, like \"<<EOT\".",
222 Subject: &tok.Range, 237 Subject: &tok.Range,
223 }) 238 })
224 239 }
240 if toldBacktick <= 2 {
225 toldBacktick++ 241 toldBacktick++
226 } 242 }
243 case TokenApostrophe:
244 if (toldApostrophe % 2) == 0 {
245 newDiag := &hcl.Diagnostic{
246 Severity: hcl.DiagError,
247 Summary: "Invalid character",
248 Detail: "Single quotes are not valid. Use double quotes (\") to enclose strings.",
249 Subject: &tok.Range,
250 }
251 diags = append(diags, newDiag)
252 }
253 if toldApostrophe <= 2 {
254 toldApostrophe++
255 }
227 case TokenSemicolon: 256 case TokenSemicolon:
228 if toldSemicolon < 1 { 257 if toldSemicolon < 1 {
229 diags = append(diags, &hcl.Diagnostic{ 258 diags = append(diags, &hcl.Diagnostic{
230 Severity: hcl.DiagError, 259 Severity: hcl.DiagError,
231 Summary: "Invalid character", 260 Summary: "Invalid character",
232 Detail: "The \";\" character is not valid. Use newlines to separate attributes and blocks, and commas to separate items in collection values.", 261 Detail: "The \";\" character is not valid. Use newlines to separate arguments and blocks, and commas to separate items in collection values.",
233 Subject: &tok.Range, 262 Subject: &tok.Range,
234 }) 263 })
235 264
@@ -257,6 +286,13 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics {
257 286
258 toldBadUTF8++ 287 toldBadUTF8++
259 } 288 }
289 case TokenQuotedNewline:
290 diags = append(diags, &hcl.Diagnostic{
291 Severity: hcl.DiagError,
292 Summary: "Invalid multi-line string",
293 Detail: "Quoted strings may not be split over multiple lines. To produce a multi-line string, either use the \\n escape to represent a newline character or use the \"heredoc\" multi-line template syntax.",
294 Subject: &tok.Range,
295 })
260 case TokenInvalid: 296 case TokenInvalid:
261 diags = append(diags, &hcl.Diagnostic{ 297 diags = append(diags, &hcl.Diagnostic{
262 Severity: hcl.DiagError, 298 Severity: hcl.DiagError,
@@ -264,9 +300,21 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics {
264 Detail: "This character is not used within the language.", 300 Detail: "This character is not used within the language.",
265 Subject: &tok.Range, 301 Subject: &tok.Range,
266 }) 302 })
267
268 toldTabs++
269 } 303 }
270 } 304 }
271 return diags 305 return diags
272} 306}
307
308var utf8BOM = []byte{0xef, 0xbb, 0xbf}
309
310// stripUTF8BOM checks whether the given buffer begins with a UTF-8 byte order
311// mark (0xEF 0xBB 0xBF) and, if so, returns a truncated slice with the same
312// backing array but with the BOM skipped.
313//
314// If there is no BOM present, the given slice is returned verbatim.
315func stripUTF8BOM(src []byte) []byte {
316 if bytes.HasPrefix(src, utf8BOM) {
317 return src[3:]
318 }
319 return src
320}