1 files changed, 307 insertions, 0 deletions

diff --git a/vendor/github.com/apparentlymart/go-textseg/textseg/make_tables.go b/vendor/github.com/apparentlymart/go-textseg/textseg/make_tables.go
new file mode 100644
index 0000000..aad3d05
--- /dev/null
+++ b/vendor/github.com/apparentlymart/go-textseg/textseg/make_tables.go
@@ -0,0 +1,307 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// Modified by Martin Atkins to serve the needs of package textseg.
+
+// +build ignore
+
+package main
+
+import (
+        "bufio"
+        "flag"
+        "fmt"
+        "io"
+        "log"
+        "net/http"
+        "os"
+        "os/exec"
+        "sort"
+        "strconv"
+        "strings"
+        "unicode"
+)
+
+var url = flag.String("url",
+        "http://www.unicode.org/Public/"+unicode.Version+"/ucd/auxiliary/",
+        "URL of Unicode database directory")
+var verbose = flag.Bool("verbose",
+        false,
+        "write data to stdout as it is parsed")
+var localFiles = flag.Bool("local",
+        false,
+        "data files have been copied to the current directory; for debugging only")
+var outputFile = flag.String("output",
+        "",
+        "output file for generated tables; default stdout")
+
+var output *bufio.Writer
+
+func main() {
+        flag.Parse()
+        setupOutput()
+
+        graphemePropertyRanges := make(map[string]*unicode.RangeTable)
+        loadUnicodeData("GraphemeBreakProperty.txt", graphemePropertyRanges)
+        wordPropertyRanges := make(map[string]*unicode.RangeTable)
+        loadUnicodeData("WordBreakProperty.txt", wordPropertyRanges)
+        sentencePropertyRanges := make(map[string]*unicode.RangeTable)
+        loadUnicodeData("SentenceBreakProperty.txt", sentencePropertyRanges)
+
+        fmt.Fprintf(output, fileHeader, *url)
+        generateTables("Grapheme", graphemePropertyRanges)
+        generateTables("Word", wordPropertyRanges)
+        generateTables("Sentence", sentencePropertyRanges)
+
+        flushOutput()
+}
+
+// WordBreakProperty.txt has the form:
+// 05F0..05F2    ; Hebrew_Letter # Lo   [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD
+// FB1D          ; Hebrew_Letter # Lo       HEBREW LETTER YOD WITH HIRIQ
+func openReader(file string) (input io.ReadCloser) {
+        if *localFiles {
+                f, err := os.Open(file)
+                if err != nil {
+                        log.Fatal(err)
+                }
+                input = f
+        } else {
+                path := *url + file
+                resp, err := http.Get(path)
+                if err != nil {
+                        log.Fatal(err)
+                }
+                if resp.StatusCode != 200 {
+                        log.Fatal("bad GET status for "+file, resp.Status)
+                }
+                input = resp.Body
+        }
+        return
+}
+
+func loadUnicodeData(filename string, propertyRanges map[string]*unicode.RangeTable) {
+        f := openReader(filename)
+        defer f.Close()
+        bufioReader := bufio.NewReader(f)
+        line, err := bufioReader.ReadString('\n')
+        for err == nil {
+                parseLine(line, propertyRanges)
+                line, err = bufioReader.ReadString('\n')
+        }
+        // if the err was EOF still need to process last value
+        if err == io.EOF {
+                parseLine(line, propertyRanges)
+        }
+}
+
+const comment = "#"
+const sep = ";"
+const rnge = ".."
+
+func parseLine(line string, propertyRanges map[string]*unicode.RangeTable) {
+        if strings.HasPrefix(line, comment) {
+                return
+        }
+        line = strings.TrimSpace(line)
+        if len(line) == 0 {
+                return
+        }
+        commentStart := strings.Index(line, comment)
+        if commentStart > 0 {
+                line = line[0:commentStart]
+        }
+        pieces := strings.Split(line, sep)
+        if len(pieces) != 2 {
+                log.Printf("unexpected %d pieces in %s", len(pieces), line)
+                return
+        }
+
+        propertyName := strings.TrimSpace(pieces[1])
+
+        rangeTable, ok := propertyRanges[propertyName]
+        if !ok {
+                rangeTable = &unicode.RangeTable{
+                        LatinOffset: 0,
+                }
+                propertyRanges[propertyName] = rangeTable
+        }
+
+        codepointRange := strings.TrimSpace(pieces[0])
+        rngeIndex := strings.Index(codepointRange, rnge)
+
+        if rngeIndex < 0 {
+                // single codepoint, not range
+                codepointInt, err := strconv.ParseUint(codepointRange, 16, 64)
+                if err != nil {
+                        log.Printf("error parsing int: %v", err)
+                        return
+                }
+                if codepointInt < 0x10000 {
+                        r16 := unicode.Range16{
+                                Lo:     uint16(codepointInt),
+                                Hi:     uint16(codepointInt),
+                                Stride: 1,
+                        }
+                        addR16ToTable(rangeTable, r16)
+                } else {
+                        r32 := unicode.Range32{
+                                Lo:     uint32(codepointInt),
+                                Hi:     uint32(codepointInt),
+                                Stride: 1,
+                        }
+                        addR32ToTable(rangeTable, r32)
+                }
+        } else {
+                rngeStart := codepointRange[0:rngeIndex]
+                rngeEnd := codepointRange[rngeIndex+2:]
+                rngeStartInt, err := strconv.ParseUint(rngeStart, 16, 64)
+                if err != nil {
+                        log.Printf("error parsing int: %v", err)
+                        return
+                }
+                rngeEndInt, err := strconv.ParseUint(rngeEnd, 16, 64)
+                if err != nil {
+                        log.Printf("error parsing int: %v", err)
+                        return
+                }
+                if rngeStartInt < 0x10000 && rngeEndInt < 0x10000 {
+                        r16 := unicode.Range16{
+                                Lo:     uint16(rngeStartInt),
+                                Hi:     uint16(rngeEndInt),
+                                Stride: 1,
+                        }
+                        addR16ToTable(rangeTable, r16)
+                } else if rngeStartInt >= 0x10000 && rngeEndInt >= 0x10000 {
+                        r32 := unicode.Range32{
+                                Lo:     uint32(rngeStartInt),
+                                Hi:     uint32(rngeEndInt),
+                                Stride: 1,
+                        }
+                        addR32ToTable(rangeTable, r32)
+                } else {
+                        log.Printf("unexpected range")
+                }
+        }
+}
+
+func addR16ToTable(r *unicode.RangeTable, r16 unicode.Range16) {
+        if r.R16 == nil {
+                r.R16 = make([]unicode.Range16, 0, 1)
+        }
+        r.R16 = append(r.R16, r16)
+        if r16.Hi <= unicode.MaxLatin1 {
+                r.LatinOffset++
+        }
+}
+
+func addR32ToTable(r *unicode.RangeTable, r32 unicode.Range32) {
+        if r.R32 == nil {
+                r.R32 = make([]unicode.Range32, 0, 1)
+        }
+        r.R32 = append(r.R32, r32)
+}
+
+func generateTables(prefix string, propertyRanges map[string]*unicode.RangeTable) {
+        prNames := make([]string, 0, len(propertyRanges))
+        for k := range propertyRanges {
+                prNames = append(prNames, k)
+        }
+        sort.Strings(prNames)
+        for _, key := range prNames {
+                rt := propertyRanges[key]
+                fmt.Fprintf(output, "var _%s%s = %s\n", prefix, key, generateRangeTable(rt))
+        }
+        fmt.Fprintf(output, "type _%sRuneRange unicode.RangeTable\n", prefix)
+
+        fmt.Fprintf(output, "func _%sRuneType(r rune) *_%sRuneRange {\n", prefix, prefix)
+        fmt.Fprintf(output, "\tswitch {\n")
+        for _, key := range prNames {
+                fmt.Fprintf(output, "\tcase unicode.Is(_%s%s, r):\n\t\treturn (*_%sRuneRange)(_%s%s)\n", prefix, key, prefix, prefix, key)
+        }
+        fmt.Fprintf(output, "\tdefault:\n\t\treturn nil\n")
+        fmt.Fprintf(output, "\t}\n")
+        fmt.Fprintf(output, "}\n")
+
+        fmt.Fprintf(output, "func (rng *_%sRuneRange) String() string {\n", prefix)
+        fmt.Fprintf(output, "\tswitch (*unicode.RangeTable)(rng) {\n")
+        for _, key := range prNames {
+                fmt.Fprintf(output, "\tcase _%s%s:\n\t\treturn %q\n", prefix, key, key)
+        }
+        fmt.Fprintf(output, "\tdefault:\n\t\treturn \"Other\"\n")
+        fmt.Fprintf(output, "\t}\n")
+        fmt.Fprintf(output, "}\n")
+}
+
+func generateRangeTable(rt *unicode.RangeTable) string {
+        rv := "&unicode.RangeTable{\n"
+        if rt.R16 != nil {
+                rv += "\tR16: []unicode.Range16{\n"
+                for _, r16 := range rt.R16 {
+                        rv += fmt.Sprintf("\t\t%#v,\n", r16)
+                }
+                rv += "\t},\n"
+        }
+        if rt.R32 != nil {
+                rv += "\tR32: []unicode.Range32{\n"
+                for _, r32 := range rt.R32 {
+                        rv += fmt.Sprintf("\t\t%#v,\n", r32)
+                }
+                rv += "\t},\n"
+        }
+        rv += fmt.Sprintf("\t\tLatinOffset: %d,\n", rt.LatinOffset)
+        rv += "}\n"
+        return rv
+}
+
+const fileHeader = `// Generated by running
+//      maketables --url=%s
+// DO NOT EDIT
+
+package textseg
+
+import(
+        "unicode"
+)
+`
+
+func setupOutput() {
+        output = bufio.NewWriter(startGofmt())
+}
+
+// startGofmt connects output to a gofmt process if -output is set.
+func startGofmt() io.Writer {
+        if *outputFile == "" {
+                return os.Stdout
+        }
+        stdout, err := os.Create(*outputFile)
+        if err != nil {
+                log.Fatal(err)
+        }
+        // Pipe output to gofmt.
+        gofmt := exec.Command("gofmt")
+        fd, err := gofmt.StdinPipe()
+        if err != nil {
+                log.Fatal(err)
+        }
+        gofmt.Stdout = stdout
+        gofmt.Stderr = os.Stderr
+        err = gofmt.Start()
+        if err != nil {
+                log.Fatal(err)
+        }
+        return fd
+}
+
+func flushOutput() {
+        err := output.Flush()
+        if err != nil {
+                log.Fatal(err)
+        }
+}