vendor/github.com/hashicorp/hcl2/hclwrite/tokens.go

   1 package hclwrite
   2
   3 import (
   4         "bytes"
   5         "io"
   6
   7         "github.com/apparentlymart/go-textseg/textseg"
   8         "github.com/hashicorp/hcl2/hcl"
   9         "github.com/hashicorp/hcl2/hcl/hclsyntax"
  10 )
  11
  12 // Token is a single sequence of bytes annotated with a type. It is similar
  13 // in purpose to hclsyntax.Token, but discards the source position information
  14 // since that is not useful in code generation.
  15 type Token struct {
  16         Type  hclsyntax.TokenType
  17         Bytes []byte
  18
  19         // We record the number of spaces before each token so that we can
  20         // reproduce the exact layout of the original file when we're making
  21         // surgical changes in-place. When _new_ code is created it will always
  22         // be in the canonical style, but we preserve layout of existing code.
  23         SpacesBefore int
  24 }
  25
  26 // asHCLSyntax returns the receiver expressed as an incomplete hclsyntax.Token.
  27 // A complete token is not possible since we don't have source location
  28 // information here, and so this method is unexported so we can be sure it will
  29 // only be used for internal purposes where we know the range isn't important.
  30 //
  31 // This is primarily intended to allow us to re-use certain functionality from
  32 // hclsyntax rather than re-implementing it against our own token type here.
  33 func (t *Token) asHCLSyntax() hclsyntax.Token {
  34         return hclsyntax.Token{
  35                 Type:  t.Type,
  36                 Bytes: t.Bytes,
  37                 Range: hcl.Range{
  38                         Filename: "<invalid>",
  39                 },
  40         }
  41 }
  42
  43 // Tokens is a flat list of tokens.
  44 type Tokens []*Token
  45
  46 func (ts Tokens) Bytes() []byte {
  47         buf := &bytes.Buffer{}
  48         ts.WriteTo(buf)
  49         return buf.Bytes()
  50 }
  51
  52 func (ts Tokens) testValue() string {
  53         return string(ts.Bytes())
  54 }
  55
  56 // Columns returns the number of columns (grapheme clusters) the token sequence
  57 // occupies. The result is not meaningful if there are newline or single-line
  58 // comment tokens in the sequence.
  59 func (ts Tokens) Columns() int {
  60         ret := 0
  61         for _, token := range ts {
  62                 ret += token.SpacesBefore // spaces are always worth one column each
  63                 ct, _ := textseg.TokenCount(token.Bytes, textseg.ScanGraphemeClusters)
  64                 ret += ct
  65         }
  66         return ret
  67 }
  68
  69 // WriteTo takes an io.Writer and writes the bytes for each token to it,
  70 // along with the spacing that separates each token. In other words, this
  71 // allows serializing the tokens to a file or other such byte stream.
  72 func (ts Tokens) WriteTo(wr io.Writer) (int64, error) {
  73         // We know we're going to be writing a lot of small chunks of repeated
  74         // space characters, so we'll prepare a buffer of these that we can
  75         // easily pass to wr.Write without any further allocation.
  76         spaces := make([]byte, 40)
  77         for i := range spaces {
  78                 spaces[i] = ' '
  79         }
  80
  81         var n int64
  82         var err error
  83         for _, token := range ts {
  84                 if err != nil {
  85                         return n, err
  86                 }
  87
  88                 for spacesBefore := token.SpacesBefore; spacesBefore > 0; spacesBefore -= len(spaces) {
  89                         thisChunk := spacesBefore
  90                         if thisChunk > len(spaces) {
  91                                 thisChunk = len(spaces)
  92                         }
  93                         var thisN int
  94                         thisN, err = wr.Write(spaces[:thisChunk])
  95                         n += int64(thisN)
  96                         if err != nil {
  97                                 return n, err
  98                         }
  99                 }
 100
 101                 var thisN int
 102                 thisN, err = wr.Write(token.Bytes)
 103                 n += int64(thisN)
 104         }
 105
 106         return n, err
 107 }
 108
 109 func (ts Tokens) walkChildNodes(w internalWalkFunc) {
 110         // Unstructured tokens have no child nodes
 111 }
 112
 113 func (ts Tokens) BuildTokens(to Tokens) Tokens {
 114         return append(to, ts...)
 115 }
 116
 117 func newIdentToken(name string) *Token {
 118         return &Token{
 119                 Type:  hclsyntax.TokenIdent,
 120                 Bytes: []byte(name),
 121         }
 122 }