1 files changed, 30 insertions, 0 deletions

diff --git a/vendor/github.com/apparentlymart/go-textseg/textseg/all_tokens.go b/vendor/github.com/apparentlymart/go-textseg/textseg/all_tokens.go
new file mode 100644
index 0000000..5752e9e
--- /dev/null
+++ b/vendor/github.com/apparentlymart/go-textseg/textseg/all_tokens.go
@@ -0,0 +1,30 @@
+package textseg
+
+import (
+        "bufio"
+        "bytes"
+)
+
+// AllTokens is a utility that uses a bufio.SplitFunc to produce a slice of
+// all of the recognized tokens in the given buffer.
+func AllTokens(buf []byte, splitFunc bufio.SplitFunc) ([][]byte, error) {
+        scanner := bufio.NewScanner(bytes.NewReader(buf))
+        scanner.Split(splitFunc)
+        var ret [][]byte
+        for scanner.Scan() {
+                ret = append(ret, scanner.Bytes())
+        }
+        return ret, scanner.Err()
+}
+
+// TokenCount is a utility that uses a bufio.SplitFunc to count the number of
+// recognized tokens in the given buffer.
+func TokenCount(buf []byte, splitFunc bufio.SplitFunc) (int, error) {
+        scanner := bufio.NewScanner(bytes.NewReader(buf))
+        scanner.Split(splitFunc)
+        var ret int
+        for scanner.Scan() {
+                ret++
+        }
+        return ret, scanner.Err()
+}