diff options
Diffstat (limited to 'vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go')
-rw-r--r-- | vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go | 92 |
1 files changed, 70 insertions, 22 deletions
diff --git a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go index bcaa15f..3d898fd 100644 --- a/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go +++ b/vendor/github.com/hashicorp/hcl2/hcl/hclsyntax/token.go | |||
@@ -1,6 +1,7 @@ | |||
1 | package hclsyntax | 1 | package hclsyntax |
2 | 2 | ||
3 | import ( | 3 | import ( |
4 | "bytes" | ||
4 | "fmt" | 5 | "fmt" |
5 | 6 | ||
6 | "github.com/apparentlymart/go-textseg/textseg" | 7 | "github.com/apparentlymart/go-textseg/textseg" |
@@ -84,16 +85,18 @@ const ( | |||
84 | // things that might work in other languages they are familiar with, or | 85 | // things that might work in other languages they are familiar with, or |
85 | // simply make incorrect assumptions about the HCL language. | 86 | // simply make incorrect assumptions about the HCL language. |
86 | 87 | ||
87 | TokenBitwiseAnd TokenType = '&' | 88 | TokenBitwiseAnd TokenType = '&' |
88 | TokenBitwiseOr TokenType = '|' | 89 | TokenBitwiseOr TokenType = '|' |
89 | TokenBitwiseNot TokenType = '~' | 90 | TokenBitwiseNot TokenType = '~' |
90 | TokenBitwiseXor TokenType = '^' | 91 | TokenBitwiseXor TokenType = '^' |
91 | TokenStarStar TokenType = '➚' | 92 | TokenStarStar TokenType = '➚' |
92 | TokenBacktick TokenType = '`' | 93 | TokenApostrophe TokenType = '\'' |
93 | TokenSemicolon TokenType = ';' | 94 | TokenBacktick TokenType = '`' |
94 | TokenTabs TokenType = '␉' | 95 | TokenSemicolon TokenType = ';' |
95 | TokenInvalid TokenType = '�' | 96 | TokenTabs TokenType = '␉' |
96 | TokenBadUTF8 TokenType = '💩' | 97 | TokenInvalid TokenType = '�' |
98 | TokenBadUTF8 TokenType = '💩' | ||
99 | TokenQuotedNewline TokenType = '' | ||
97 | 100 | ||
98 | // TokenNil is a placeholder for when a token is required but none is | 101 | // TokenNil is a placeholder for when a token is required but none is |
99 | // available, e.g. when reporting errors. The scanner will never produce | 102 | // available, e.g. when reporting errors. The scanner will never produce |
@@ -114,10 +117,11 @@ const ( | |||
114 | ) | 117 | ) |
115 | 118 | ||
116 | type tokenAccum struct { | 119 | type tokenAccum struct { |
117 | Filename string | 120 | Filename string |
118 | Bytes []byte | 121 | Bytes []byte |
119 | Pos hcl.Pos | 122 | Pos hcl.Pos |
120 | Tokens []Token | 123 | Tokens []Token |
124 | StartByte int | ||
121 | } | 125 | } |
122 | 126 | ||
123 | func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) { | 127 | func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) { |
@@ -125,11 +129,11 @@ func (f *tokenAccum) emitToken(ty TokenType, startOfs, endOfs int) { | |||
125 | // the start pos to get our end pos. | 129 | // the start pos to get our end pos. |
126 | 130 | ||
127 | start := f.Pos | 131 | start := f.Pos |
128 | start.Column += startOfs - f.Pos.Byte // Safe because only ASCII spaces can be in the offset | 132 | start.Column += startOfs + f.StartByte - f.Pos.Byte // Safe because only ASCII spaces can be in the offset |
129 | start.Byte = startOfs | 133 | start.Byte = startOfs + f.StartByte |
130 | 134 | ||
131 | end := start | 135 | end := start |
132 | end.Byte = endOfs | 136 | end.Byte = endOfs + f.StartByte |
133 | b := f.Bytes[startOfs:endOfs] | 137 | b := f.Bytes[startOfs:endOfs] |
134 | for len(b) > 0 { | 138 | for len(b) > 0 { |
135 | advance, seq, _ := textseg.ScanGraphemeClusters(b, true) | 139 | advance, seq, _ := textseg.ScanGraphemeClusters(b, true) |
@@ -160,6 +164,13 @@ type heredocInProgress struct { | |||
160 | StartOfLine bool | 164 | StartOfLine bool |
161 | } | 165 | } |
162 | 166 | ||
167 | func tokenOpensFlushHeredoc(tok Token) bool { | ||
168 | if tok.Type != TokenOHeredoc { | ||
169 | return false | ||
170 | } | ||
171 | return bytes.HasPrefix(tok.Bytes, []byte{'<', '<', '-'}) | ||
172 | } | ||
173 | |||
163 | // checkInvalidTokens does a simple pass across the given tokens and generates | 174 | // checkInvalidTokens does a simple pass across the given tokens and generates |
164 | // diagnostics for tokens that should _never_ appear in HCL source. This | 175 | // diagnostics for tokens that should _never_ appear in HCL source. This |
165 | // is intended to avoid the need for the parser to have special support | 176 | // is intended to avoid the need for the parser to have special support |
@@ -174,11 +185,15 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics { | |||
174 | toldBitwise := 0 | 185 | toldBitwise := 0 |
175 | toldExponent := 0 | 186 | toldExponent := 0 |
176 | toldBacktick := 0 | 187 | toldBacktick := 0 |
188 | toldApostrophe := 0 | ||
177 | toldSemicolon := 0 | 189 | toldSemicolon := 0 |
178 | toldTabs := 0 | 190 | toldTabs := 0 |
179 | toldBadUTF8 := 0 | 191 | toldBadUTF8 := 0 |
180 | 192 | ||
181 | for _, tok := range tokens { | 193 | for _, tok := range tokens { |
194 | // copy token so it's safe to point to it | ||
195 | tok := tok | ||
196 | |||
182 | switch tok.Type { | 197 | switch tok.Type { |
183 | case TokenBitwiseAnd, TokenBitwiseOr, TokenBitwiseXor, TokenBitwiseNot: | 198 | case TokenBitwiseAnd, TokenBitwiseOr, TokenBitwiseXor, TokenBitwiseNot: |
184 | if toldBitwise < 4 { | 199 | if toldBitwise < 4 { |
@@ -214,22 +229,36 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics { | |||
214 | case TokenBacktick: | 229 | case TokenBacktick: |
215 | // Only report for alternating (even) backticks, so we won't report both start and ends of the same | 230 | // Only report for alternating (even) backticks, so we won't report both start and ends of the same |
216 | // backtick-quoted string. | 231 | // backtick-quoted string. |
217 | if toldExponent < 4 && (toldExponent%2) == 0 { | 232 | if (toldBacktick % 2) == 0 { |
218 | diags = append(diags, &hcl.Diagnostic{ | 233 | diags = append(diags, &hcl.Diagnostic{ |
219 | Severity: hcl.DiagError, | 234 | Severity: hcl.DiagError, |
220 | Summary: "Invalid character", | 235 | Summary: "Invalid character", |
221 | Detail: "The \"`\" character is not valid. To create a multi-line string, use the \"heredoc\" syntax, like \"<<EOT\".", | 236 | Detail: "The \"`\" character is not valid. To create a multi-line string, use the \"heredoc\" syntax, like \"<<EOT\".", |
222 | Subject: &tok.Range, | 237 | Subject: &tok.Range, |
223 | }) | 238 | }) |
224 | 239 | } | |
240 | if toldBacktick <= 2 { | ||
225 | toldBacktick++ | 241 | toldBacktick++ |
226 | } | 242 | } |
243 | case TokenApostrophe: | ||
244 | if (toldApostrophe % 2) == 0 { | ||
245 | newDiag := &hcl.Diagnostic{ | ||
246 | Severity: hcl.DiagError, | ||
247 | Summary: "Invalid character", | ||
248 | Detail: "Single quotes are not valid. Use double quotes (\") to enclose strings.", | ||
249 | Subject: &tok.Range, | ||
250 | } | ||
251 | diags = append(diags, newDiag) | ||
252 | } | ||
253 | if toldApostrophe <= 2 { | ||
254 | toldApostrophe++ | ||
255 | } | ||
227 | case TokenSemicolon: | 256 | case TokenSemicolon: |
228 | if toldSemicolon < 1 { | 257 | if toldSemicolon < 1 { |
229 | diags = append(diags, &hcl.Diagnostic{ | 258 | diags = append(diags, &hcl.Diagnostic{ |
230 | Severity: hcl.DiagError, | 259 | Severity: hcl.DiagError, |
231 | Summary: "Invalid character", | 260 | Summary: "Invalid character", |
232 | Detail: "The \";\" character is not valid. Use newlines to separate attributes and blocks, and commas to separate items in collection values.", | 261 | Detail: "The \";\" character is not valid. Use newlines to separate arguments and blocks, and commas to separate items in collection values.", |
233 | Subject: &tok.Range, | 262 | Subject: &tok.Range, |
234 | }) | 263 | }) |
235 | 264 | ||
@@ -257,6 +286,13 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics { | |||
257 | 286 | ||
258 | toldBadUTF8++ | 287 | toldBadUTF8++ |
259 | } | 288 | } |
289 | case TokenQuotedNewline: | ||
290 | diags = append(diags, &hcl.Diagnostic{ | ||
291 | Severity: hcl.DiagError, | ||
292 | Summary: "Invalid multi-line string", | ||
293 | Detail: "Quoted strings may not be split over multiple lines. To produce a multi-line string, either use the \\n escape to represent a newline character or use the \"heredoc\" multi-line template syntax.", | ||
294 | Subject: &tok.Range, | ||
295 | }) | ||
260 | case TokenInvalid: | 296 | case TokenInvalid: |
261 | diags = append(diags, &hcl.Diagnostic{ | 297 | diags = append(diags, &hcl.Diagnostic{ |
262 | Severity: hcl.DiagError, | 298 | Severity: hcl.DiagError, |
@@ -264,9 +300,21 @@ func checkInvalidTokens(tokens Tokens) hcl.Diagnostics { | |||
264 | Detail: "This character is not used within the language.", | 300 | Detail: "This character is not used within the language.", |
265 | Subject: &tok.Range, | 301 | Subject: &tok.Range, |
266 | }) | 302 | }) |
267 | |||
268 | toldTabs++ | ||
269 | } | 303 | } |
270 | } | 304 | } |
271 | return diags | 305 | return diags |
272 | } | 306 | } |
307 | |||
308 | var utf8BOM = []byte{0xef, 0xbb, 0xbf} | ||
309 | |||
310 | // stripUTF8BOM checks whether the given buffer begins with a UTF-8 byte order | ||
311 | // mark (0xEF 0xBB 0xBF) and, if so, returns a truncated slice with the same | ||
312 | // backing array but with the BOM skipped. | ||
313 | // | ||
314 | // If there is no BOM present, the given slice is returned verbatim. | ||
315 | func stripUTF8BOM(src []byte) []byte { | ||
316 | if bytes.HasPrefix(src, utf8BOM) { | ||
317 | return src[3:] | ||
318 | } | ||
319 | return src | ||
320 | } | ||