From 15c0b25d011f37e7c20aeca9eaf461f78285b8d9 Mon Sep 17 00:00:00 2001 From: Alex Pilon Date: Fri, 22 Feb 2019 18:24:37 -0500 Subject: deps: github.com/hashicorp/terraform@sdk-v0.11-with-go-modules Updated via: go get github.com/hashicorp/terraform@sdk-v0.11-with-go-modules and go mod tidy --- .../go-textseg/textseg/all_tokens.go | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 vendor/github.com/apparentlymart/go-textseg/textseg/all_tokens.go (limited to 'vendor/github.com/apparentlymart/go-textseg/textseg/all_tokens.go') diff --git a/vendor/github.com/apparentlymart/go-textseg/textseg/all_tokens.go b/vendor/github.com/apparentlymart/go-textseg/textseg/all_tokens.go new file mode 100644 index 0000000..5752e9e --- /dev/null +++ b/vendor/github.com/apparentlymart/go-textseg/textseg/all_tokens.go @@ -0,0 +1,30 @@ +package textseg + +import ( + "bufio" + "bytes" +) + +// AllTokens is a utility that uses a bufio.SplitFunc to produce a slice of +// all of the recognized tokens in the given buffer. +func AllTokens(buf []byte, splitFunc bufio.SplitFunc) ([][]byte, error) { + scanner := bufio.NewScanner(bytes.NewReader(buf)) + scanner.Split(splitFunc) + var ret [][]byte + for scanner.Scan() { + ret = append(ret, scanner.Bytes()) + } + return ret, scanner.Err() +} + +// TokenCount is a utility that uses a bufio.SplitFunc to count the number of +// recognized tokens in the given buffer. +func TokenCount(buf []byte, splitFunc bufio.SplitFunc) (int, error) { + scanner := bufio.NewScanner(bytes.NewReader(buf)) + scanner.Split(splitFunc) + var ret int + for scanner.Scan() { + ret++ + } + return ret, scanner.Err() +} -- cgit v1.2.3