7 // isWhitespace will return whether or not the character is
8 // a whitespace character.
10 // Whitespace is defined as a space or tab.
11 func isWhitespace(c rune) bool {
12 return unicode.IsSpace(c) && c != '\n' && c != '\r'
15 func newWSToken(b []rune) (Token, int, error) {
17 for ; i < len(b); i++ {
18 if !isWhitespace(b[i]) {
23 return newToken(TokenWS, b[:i], NoneType), i, nil