diff --git a/gen_testdata.go b/gen_testdata.go
new file mode 100644
index 0000000000000000000000000000000000000000..fed4aaf468cc95e4dcefd1a662808649de50fa1f
--- /dev/null
+++ b/gen_testdata.go
@@ -0,0 +1,56 @@
+// +build ignore
+
+// gen_testdata clones the utf-8 tests data to the other
+// unicode encodings and adds BOM variants of each.
+package main
+
+import (
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/unicode"
+)
+
+func main() {
+ var xforms = []struct {
+ dir, bom string
+ enc encoding.Encoding
+ }{
+ {"testdata/utf-16be", "\xFE\xFF", unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)},
+ {"testdata/utf-16le", "\xFF\xFE", unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)},
+ }
+
+ paths, _ := filepath.Glob("testdata/utf-8/*")
+ for _, p := range paths {
+ src, err := ioutil.ReadFile(p)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ write("testdata/utf-8_bom", p, "\xEF\xBB\xBF", src)
+ for _, xform := range xforms {
+ dst, err := xform.enc.NewEncoder().Bytes(src)
+ if err != nil {
+ log.Fatal(err)
+ }
+ write(xform.dir, p, "", dst)
+ write(xform.dir+"_bom", p, xform.bom, dst)
+ }
+ }
+}
+
+func write(dir, orig, bom string, buf []byte) {
+ f, err := os.Create(filepath.Join(dir, filepath.Base(orig)))
+ if err != nil {
+ log.Fatal(err)
+ }
+ if _, err = f.Write([]byte(bom)); err != nil {
+ log.Fatal(err)
+ }
+ if _, err = f.Write(buf); err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/main.go b/main.go
index 8ee176722d202dd82842a37e424ba339d9566d12..bf3344106b50d099c4a68c695bac92c5c5db9650 100644
--- a/main.go
+++ b/main.go
@@ -2,8 +2,11 @@
// a provided JSON Schema - https://json-schema.org/
package main
+//go:generate go run gen_testdata.go
+
import (
"bufio"
+ "bytes"
"flag"
"fmt"
"io"
@@ -15,21 +18,37 @@ import (
"strings"
"sync"
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/unicode"
+
"github.com/ghodss/yaml"
"github.com/mitchellh/go-homedir"
"github.com/xeipuuv/gojsonschema"
)
var (
- version = "v1.3.0-dev"
+ version = "v1.4.0-dev"
schemaFlag = flag.String("s", "", "primary JSON schema to validate against, required")
quietFlag = flag.Bool("q", false, "quiet, only print validation failures and errors")
versionFlag = flag.Bool("v", false, "print version and exit")
+ bomFlag = flag.Bool("b", false, "allow BOM in JSON files, error if seen and unset")
listFlags stringFlags
refFlags stringFlags
)
+// https://en.wikipedia.org/wiki/Byte_order_mark#Byte_order_marks_by_encoding
+const (
+ bomUTF8 = "\xEF\xBB\xBF"
+ bomUTF16BE = "\xFE\xFF"
+ bomUTF16LE = "\xFF\xFE"
+)
+
+var (
+ encUTF16BE = unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)
+ encUTF16LE = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)
+)
+
func init() {
flag.Var(&listFlags, "l", "validate JSON documents from newline separated paths and/or globs in a text file (relative to the basename of the file itself)")
flag.Var(&refFlags, "r", "referenced schema(s), can be globs and/or used multiple times")
@@ -60,7 +79,7 @@ func realMain(args []string, w io.Writer) int {
dir := filepath.Dir(list)
f, err := os.Open(list)
if err != nil {
- log.Fatalf("%s: %s\n", list, err)
+ return schemaError("%s: %s", list, err)
}
defer f.Close()
@@ -74,7 +93,7 @@ func realMain(args []string, w io.Writer) int {
docs = append(docs, glob(pattern)...)
}
if err := scanner.Err(); err != nil {
- log.Fatalf("%s: invalid file list: %s\n", list, err)
+ return schemaError("%s: invalid file list: %s", list, err)
}
}
if len(docs) == 0 {
@@ -85,13 +104,13 @@ func realMain(args []string, w io.Writer) int {
sl := gojsonschema.NewSchemaLoader()
schemaPath, err := filepath.Abs(*schemaFlag)
if err != nil {
- log.Fatalf("%s: unable to convert to absolute path: %s\n", *schemaFlag, err)
+ return schemaError("%s: unable to convert to absolute path: %s", *schemaFlag, err)
}
for _, ref := range refFlags {
for _, p := range glob(ref) {
absPath, err := filepath.Abs(p)
if err != nil {
- log.Fatalf("%s: unable to convert to absolute path: %s\n", absPath, err)
+ return schemaError("%s: unable to convert to absolute path: %s", absPath, err)
}
if absPath == schemaPath {
@@ -100,22 +119,22 @@ func realMain(args []string, w io.Writer) int {
loader, err := jsonLoader(absPath)
if err != nil {
- log.Fatalf("%s: unable to load schema ref: %s\n", *schemaFlag, err)
+ return schemaError("%s: unable to load schema ref: %s", *schemaFlag, err)
}
if err := sl.AddSchemas(loader); err != nil {
- log.Fatalf("%s: invalid schema: %s\n", p, err)
+ return schemaError("%s: invalid schema: %s", p, err)
}
}
}
schemaLoader, err := jsonLoader(schemaPath)
if err != nil {
- log.Fatalf("%s: unable to load schema: %s\n", *schemaFlag, err)
+ return schemaError("%s: unable to load schema: %s", *schemaFlag, err)
}
schema, err := sl.Compile(schemaLoader)
if err != nil {
- log.Fatalf("%s: invalid schema: %s\n", *schemaFlag, err)
+ return schemaError("%s: invalid schema: %s", *schemaFlag, err)
}
// Validate the schema against each doc in parallel, limiting simultaneous
@@ -131,7 +150,6 @@ func realMain(args []string, w io.Writer) int {
sem <- 0
defer func() { <-sem }()
-
loader, err := jsonLoader(path)
if err != nil {
msg := fmt.Sprintf("%s: error: load doc: %s", path, err)
@@ -190,19 +208,62 @@ func jsonLoader(path string) (gojsonschema.JSONLoader, error) {
}
switch filepath.Ext(path) {
case ".yml", ".yaml":
+ // TODO YAML requires the precense of a BOM to detect UTF-16
+ // text. Is there a decent hueristic to detect UTF-16 text
+ // missing a BOM so we can provide a better error message?
buf, err = yaml.YAMLToJSON(buf)
+ default:
+ buf, err = jsonDecodeCharset(buf)
}
if err != nil {
return nil, err
}
+ // TODO What if we have an empty document?
return gojsonschema.NewBytesLoader(buf), nil
}
+// jsonDecodeCharset attempts to detect UTF-16 (LE or BE) JSON text and
+// decode as appropriate. It also skips a BOM at the start of the buffer
+// if `-b` was specified. Presence of a BOM is an error otherwise.
+func jsonDecodeCharset(buf []byte) ([]byte, error) {
+ if len(buf) < 2 { // UTF-8
+ return buf, nil
+ }
+
+ bom := ""
+ var enc encoding.Encoding
+ switch {
+ case bytes.HasPrefix(buf, []byte(bomUTF8)):
+ bom = bomUTF8
+ case bytes.HasPrefix(buf, []byte(bomUTF16BE)):
+ bom = bomUTF16BE
+ enc = encUTF16BE
+ case bytes.HasPrefix(buf, []byte(bomUTF16LE)):
+ bom = bomUTF16LE
+ enc = encUTF16LE
+ case buf[0] == 0:
+ enc = encUTF16BE
+ case buf[1] == 0:
+ enc = encUTF16LE
+ }
+
+ if bom != "" {
+ if !*bomFlag {
+ return nil, fmt.Errorf("unexpected BOM, see `-b` flag")
+ }
+ buf = buf[len(bom):]
+ }
+ if enc != nil {
+ return enc.NewDecoder().Bytes(buf)
+ }
+ return buf, nil
+}
+
func printUsage() {
fmt.Fprintf(os.Stderr, `Usage: %s -s schema.(json|yml) [options] document.(json|yml) ...
- yajsv validates JSON and YAML document(s) against a schema. One of three statuses are
- reported per document:
+ yajsv validates JSON and YAML document(s) against a schema. One of three status
+ results are reported per document:
pass: Document is valid relative to the schema
fail: Document is invalid relative to the schema
@@ -212,7 +273,8 @@ func printUsage() {
schema validation failure.
Sets the exit code to 1 on any failures, 2 on any errors, 3 on both, 4 on
- invalid usage. Otherwise, 0 is returned if everything passes validation.
+ invalid usage, 5 on schema definition or file-list errors. Otherwise, 0 is
+ returned if everything passes validation.
Options:
@@ -227,6 +289,11 @@ func usageError(msg string) int {
return 4
}
+func schemaError(format string, args ...interface{}) int {
+ fmt.Fprintf(os.Stderr, format+"\n", args...)
+ return 5
+}
+
// glob is a wrapper that also resolves `~` since we may be skipping
// the shell expansion when single-quoting globs at the command line
func glob(pattern string) []string {
diff --git a/main_test.go b/main_test.go
index a2883d1fe2d19fa91cd2315535b38d912c7a2d9f..42d04131ce3214075b3e4ecca3730b9ff99e43a4 100644
--- a/main_test.go
+++ b/main_test.go
@@ -1,78 +1,93 @@
package main
import (
+ "fmt"
+ "os"
"path/filepath"
"sort"
"strings"
"testing"
)
+func init() {
+ // TODO: Cleanup this global monkey-patching
+ devnull, err := os.Open(os.DevNull)
+ if err != nil {
+ panic(err)
+ }
+ os.Stderr = devnull
+}
+
func TestMain(t *testing.T) {
tests := []struct {
- in string
- out []string
+ in string
+ out []string
exit int
- } {
+ }{
{
- "-s testdata/schema.yml testdata/data-pass.yml",
- []string{"testdata/data-pass.yml: pass"},
+ "-s testdata/utf-16be_bom/schema.json testdata/utf-16le_bom/data-fail.yml",
+ []string{},
+ 5,
+ }, {
+ "-s testdata/utf-8/schema.yml testdata/utf-8/data-pass.yml",
+ []string{"testdata/utf-8/data-pass.yml: pass"},
0,
}, {
- "-s testdata/schema.json testdata/data-pass.yml",
- []string{"testdata/data-pass.yml: pass"},
+ "-s testdata/utf-8/schema.json testdata/utf-8/data-pass.yml",
+ []string{"testdata/utf-8/data-pass.yml: pass"},
0,
}, {
- "-s testdata/schema.json testdata/data-pass.json",
- []string{"testdata/data-pass.json: pass"},
+ "-s testdata/utf-8/schema.json testdata/utf-8/data-pass.json",
+ []string{"testdata/utf-8/data-pass.json: pass"},
0,
}, {
- "-s testdata/schema.yml testdata/data-pass.json",
- []string{"testdata/data-pass.json: pass"},
+ "-s testdata/utf-8/schema.yml testdata/utf-8/data-pass.json",
+ []string{"testdata/utf-8/data-pass.json: pass"},
0,
}, {
- "-q -s testdata/schema.yml testdata/data-fail.yml",
- []string{"testdata/data-fail.yml: fail: (root): foo is required"},
+ "-q -s testdata/utf-8/schema.yml testdata/utf-8/data-fail.yml",
+ []string{"testdata/utf-8/data-fail.yml: fail: (root): foo is required"},
1,
}, {
- "-q -s testdata/schema.json testdata/data-fail.yml",
- []string{"testdata/data-fail.yml: fail: (root): foo is required"},
+ "-q -s testdata/utf-8/schema.json testdata/utf-8/data-fail.yml",
+ []string{"testdata/utf-8/data-fail.yml: fail: (root): foo is required"},
1,
}, {
- "-q -s testdata/schema.json testdata/data-fail.json",
- []string{"testdata/data-fail.json: fail: (root): foo is required"},
+ "-q -s testdata/utf-8/schema.json testdata/utf-8/data-fail.json",
+ []string{"testdata/utf-8/data-fail.json: fail: (root): foo is required"},
1,
}, {
- "-q -s testdata/schema.yml testdata/data-fail.json",
- []string{"testdata/data-fail.json: fail: (root): foo is required"},
+ "-q -s testdata/utf-8/schema.yml testdata/utf-8/data-fail.json",
+ []string{"testdata/utf-8/data-fail.json: fail: (root): foo is required"},
1,
}, {
- "-q -s testdata/schema.json testdata/data-error.json",
- []string{"testdata/data-error.json: error: validate: invalid character 'o' in literal null (expecting 'u')"},
+ "-q -s testdata/utf-8/schema.json testdata/utf-8/data-error.json",
+ []string{"testdata/utf-8/data-error.json: error: validate: invalid character 'o' in literal null (expecting 'u')"},
2,
}, {
- "-q -s testdata/schema.yml testdata/data-error.yml",
- []string{"testdata/data-error.yml: error: load doc: yaml: found unexpected end of stream"},
+ "-q -s testdata/utf-8/schema.yml testdata/utf-8/data-error.yml",
+ []string{"testdata/utf-8/data-error.yml: error: load doc: yaml: found unexpected end of stream"},
2,
}, {
- "-q -s testdata/schema.json testdata/data-*.json",
+ "-q -s testdata/utf-8/schema.json testdata/utf-8/data-*.json",
[]string{
- "testdata/data-fail.json: fail: (root): foo is required",
- "testdata/data-error.json: error: validate: invalid character 'o' in literal null (expecting 'u')",
+ "testdata/utf-8/data-fail.json: fail: (root): foo is required",
+ "testdata/utf-8/data-error.json: error: validate: invalid character 'o' in literal null (expecting 'u')",
}, 3,
}, {
- "-q -s testdata/schema.yml testdata/data-*.yml",
+ "-q -s testdata/utf-8/schema.yml testdata/utf-8/data-*.yml",
[]string{
- "testdata/data-error.yml: error: load doc: yaml: found unexpected end of stream",
- "testdata/data-fail.yml: fail: (root): foo is required",
+ "testdata/utf-8/data-error.yml: error: load doc: yaml: found unexpected end of stream",
+ "testdata/utf-8/data-fail.yml: fail: (root): foo is required",
}, 3,
},
}
for _, tt := range tests {
- in := strings.ReplaceAll(tt.in, "/", string(filepath.Separator))
+ in := strings.Replace(tt.in, "/", string(filepath.Separator), -1)
sort.Strings(tt.out)
out := strings.Join(tt.out, "\n")
- out = strings.ReplaceAll(out, "/", string(filepath.Separator))
+ out = strings.Replace(out, "/", string(filepath.Separator), -1)
t.Run(in, func(t *testing.T) {
var w strings.Builder
@@ -90,3 +105,79 @@ func TestMain(t *testing.T) {
}
}
+func TestMatrix(t *testing.T) {
+ // schema.{format} {encoding}{_bom}/data-{expect}.{format}
+ type testcase struct {
+ schemaEnc, schemaFmt string
+ dataEnc, dataFmt, dataRes string
+ allowBOM bool
+ }
+
+ encodings := []string{"utf-8", "utf-16be", "utf-16le", "utf-8_bom", "utf-16be_bom", "utf-16le_bom"}
+ formats := []string{"json", "yml"}
+ results := []string{"pass", "fail", "error"}
+ tests := []testcase{}
+
+ // poor mans cartesian product
+ for _, senc := range encodings {
+ for _, sfmt := range formats {
+ for _, denc := range encodings {
+ for _, dfmt := range formats {
+ for _, dres := range results {
+ tests = append(tests, testcase{senc, sfmt, denc, dfmt, dres, false})
+ tests = append(tests, testcase{senc, sfmt, denc, dfmt, dres, true})
+ }
+ }
+ }
+ }
+ }
+
+ for _, tt := range tests {
+ schemaBOM := strings.HasSuffix(tt.schemaEnc, "_bom")
+ schema16 := strings.HasPrefix(tt.schemaEnc, "utf-16")
+ dataBOM := strings.HasSuffix(tt.dataEnc, "_bom")
+ data16 := strings.HasPrefix(tt.dataEnc, "utf-16")
+
+ schema := fmt.Sprintf("testdata/%s/schema.%s", tt.schemaEnc, tt.schemaFmt)
+ data := fmt.Sprintf("testdata/%s/data-%s.%s", tt.dataEnc, tt.dataRes, tt.dataFmt)
+ cmd := fmt.Sprintf("-s %s %s", schema, data)
+ if tt.allowBOM {
+ cmd = "-b " + cmd
+ }
+
+ t.Run(cmd, func(t *testing.T) {
+ want := 0
+ switch {
+ // Schema Errors (exit = 5)
+ // - YAML w/out BOM for UTF-16
+ // - JSON w/ BOM but missing allowBOM flag
+ case tt.schemaFmt == "yml" && !schemaBOM && schema16:
+ want = 5
+ case tt.schemaFmt == "json" && schemaBOM && !tt.allowBOM:
+ want = 5
+ // Data Errors (exit = 2)
+ // - YAML w/out BOM for UTF-16
+ // - JSON w/ BOM but missing allowBOM flag
+ // - standard malformed files (e.g. data-error)
+ case tt.dataFmt == "yml" && !dataBOM && data16:
+ want = 2
+ case tt.dataFmt == "json" && dataBOM && !tt.allowBOM:
+ want = 2
+ case tt.dataRes == "error":
+ want = 2
+ // Data Failures
+ case tt.dataRes == "fail":
+ want = 1
+ }
+
+ // TODO: Cleanup this global monkey-patching
+ *bomFlag = tt.allowBOM
+
+ var w strings.Builder
+ got := realMain(strings.Split(cmd, " "), &w)
+ if got != want {
+ t.Errorf("got(%d) != want(%d) bomflag %t", got, want, *bomFlag)
+ }
+ })
+ }
+}
diff --git a/testdata/utf-16be/data-error.json b/testdata/utf-16be/data-error.json
new file mode 100644
index 0000000000000000000000000000000000000000..74920f2f29711d04beca75a244f0ce3310c1ea19
Binary files /dev/null and b/testdata/utf-16be/data-error.json differ
diff --git a/testdata/utf-16be/data-error.yml b/testdata/utf-16be/data-error.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b0c91f19b96f9d6bd513c0b368e800395b6cbec0
Binary files /dev/null and b/testdata/utf-16be/data-error.yml differ
diff --git a/testdata/utf-16be/data-fail.json b/testdata/utf-16be/data-fail.json
new file mode 100644
index 0000000000000000000000000000000000000000..646c6c55c3ae37730d6f7c019adec120c90d43b0
Binary files /dev/null and b/testdata/utf-16be/data-fail.json differ
diff --git a/testdata/utf-16be/data-fail.yml b/testdata/utf-16be/data-fail.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b0dc72605189e83b51522bfe72e89229e0528011
Binary files /dev/null and b/testdata/utf-16be/data-fail.yml differ
diff --git a/testdata/utf-16be/data-pass.json b/testdata/utf-16be/data-pass.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0d7261160409b9cd2670c27a276460b2f5a08d5
Binary files /dev/null and b/testdata/utf-16be/data-pass.json differ
diff --git a/testdata/utf-16be/data-pass.yml b/testdata/utf-16be/data-pass.yml
new file mode 100644
index 0000000000000000000000000000000000000000..80b8d780aced3b101b73f3712b377d7872341f1e
Binary files /dev/null and b/testdata/utf-16be/data-pass.yml differ
diff --git a/testdata/utf-16be/schema.json b/testdata/utf-16be/schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed742390b4f5d72837cf95dcf3d3826140eafba5
Binary files /dev/null and b/testdata/utf-16be/schema.json differ
diff --git a/testdata/utf-16be/schema.yml b/testdata/utf-16be/schema.yml
new file mode 100644
index 0000000000000000000000000000000000000000..420e8ba4bb742f8abc77d5a95bfce16379e9040f
Binary files /dev/null and b/testdata/utf-16be/schema.yml differ
diff --git a/testdata/utf-16be_bom/data-error.json b/testdata/utf-16be_bom/data-error.json
new file mode 100644
index 0000000000000000000000000000000000000000..240156890c7692c7f7d9e5df49eeb34c01bb5a8e
Binary files /dev/null and b/testdata/utf-16be_bom/data-error.json differ
diff --git a/testdata/utf-16be_bom/data-error.yml b/testdata/utf-16be_bom/data-error.yml
new file mode 100644
index 0000000000000000000000000000000000000000..499fc64eb61610a5b5c666a0cabe8ee5847a4843
Binary files /dev/null and b/testdata/utf-16be_bom/data-error.yml differ
diff --git a/testdata/utf-16be_bom/data-fail.json b/testdata/utf-16be_bom/data-fail.json
new file mode 100644
index 0000000000000000000000000000000000000000..bf74c385e84b688ef534d86071612fa20aa19703
Binary files /dev/null and b/testdata/utf-16be_bom/data-fail.json differ
diff --git a/testdata/utf-16be_bom/data-fail.yml b/testdata/utf-16be_bom/data-fail.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f7f8d1f41cb944a238aff90bf1c9dd910d5572d3
Binary files /dev/null and b/testdata/utf-16be_bom/data-fail.yml differ
diff --git a/testdata/utf-16be_bom/data-pass.json b/testdata/utf-16be_bom/data-pass.json
new file mode 100644
index 0000000000000000000000000000000000000000..d3b34921be45a5157e095bda65bbdd8a2fade476
Binary files /dev/null and b/testdata/utf-16be_bom/data-pass.json differ
diff --git a/testdata/utf-16be_bom/data-pass.yml b/testdata/utf-16be_bom/data-pass.yml
new file mode 100644
index 0000000000000000000000000000000000000000..277d96d955afa1c42549643f587d3d60e098e880
Binary files /dev/null and b/testdata/utf-16be_bom/data-pass.yml differ
diff --git a/testdata/utf-16be_bom/schema.json b/testdata/utf-16be_bom/schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..6f743f624e4ecebc55caf778b717305509cf44c4
Binary files /dev/null and b/testdata/utf-16be_bom/schema.json differ
diff --git a/testdata/utf-16be_bom/schema.yml b/testdata/utf-16be_bom/schema.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d8cbaffeda48c1b1e1d9adbdaaaa136767a75184
Binary files /dev/null and b/testdata/utf-16be_bom/schema.yml differ
diff --git a/testdata/utf-16le/data-error.json b/testdata/utf-16le/data-error.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7ff59b25585042ad5ce4859610b6ef60bbe1b23
Binary files /dev/null and b/testdata/utf-16le/data-error.json differ
diff --git a/testdata/utf-16le/data-error.yml b/testdata/utf-16le/data-error.yml
new file mode 100644
index 0000000000000000000000000000000000000000..89121bcb737e4fe9cd908ac08625bb805ee2a393
Binary files /dev/null and b/testdata/utf-16le/data-error.yml differ
diff --git a/testdata/utf-16le/data-fail.json b/testdata/utf-16le/data-fail.json
new file mode 100644
index 0000000000000000000000000000000000000000..fad1c917597c78ee47e743e14d84d01176bd772d
Binary files /dev/null and b/testdata/utf-16le/data-fail.json differ
diff --git a/testdata/utf-16le/data-fail.yml b/testdata/utf-16le/data-fail.yml
new file mode 100644
index 0000000000000000000000000000000000000000..64a5cc187a8ee92ae9e677054cd0d1fd1c86cfdc
Binary files /dev/null and b/testdata/utf-16le/data-fail.yml differ
diff --git a/testdata/utf-16le/data-pass.json b/testdata/utf-16le/data-pass.json
new file mode 100644
index 0000000000000000000000000000000000000000..28357344bee9e5cc940a51cc7e9d7e36ba644573
Binary files /dev/null and b/testdata/utf-16le/data-pass.json differ
diff --git a/testdata/utf-16le/data-pass.yml b/testdata/utf-16le/data-pass.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ebb8581e4f4f41e334045d3dbe11af0809d2bd3d
Binary files /dev/null and b/testdata/utf-16le/data-pass.yml differ
diff --git a/testdata/utf-16le/schema.json b/testdata/utf-16le/schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..2b84fea7e19373707542feccfccd560ad2166318
Binary files /dev/null and b/testdata/utf-16le/schema.json differ
diff --git a/testdata/utf-16le/schema.yml b/testdata/utf-16le/schema.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e235f85ec6f2ba862675cae8b4bf3ec369cb134a
Binary files /dev/null and b/testdata/utf-16le/schema.yml differ
diff --git a/testdata/utf-16le_bom/data-error.json b/testdata/utf-16le_bom/data-error.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f103343818acdbcda160e0dda735ab53391ded3
Binary files /dev/null and b/testdata/utf-16le_bom/data-error.json differ
diff --git a/testdata/utf-16le_bom/data-error.yml b/testdata/utf-16le_bom/data-error.yml
new file mode 100644
index 0000000000000000000000000000000000000000..838a4a95543495aefd1a483dd541179c6ae2def2
Binary files /dev/null and b/testdata/utf-16le_bom/data-error.yml differ
diff --git a/testdata/utf-16le_bom/data-fail.json b/testdata/utf-16le_bom/data-fail.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5030d96f83b7edb35e8132b448b90acfef2c645
Binary files /dev/null and b/testdata/utf-16le_bom/data-fail.json differ
diff --git a/testdata/utf-16le_bom/data-fail.yml b/testdata/utf-16le_bom/data-fail.yml
new file mode 100644
index 0000000000000000000000000000000000000000..90d4179e98f5ddfa8b6138fd2f654542461cc7cb
Binary files /dev/null and b/testdata/utf-16le_bom/data-fail.yml differ
diff --git a/testdata/utf-16le_bom/data-pass.json b/testdata/utf-16le_bom/data-pass.json
new file mode 100644
index 0000000000000000000000000000000000000000..04c3ad0db6025affd7921db24988c2f593fc7938
Binary files /dev/null and b/testdata/utf-16le_bom/data-pass.json differ
diff --git a/testdata/utf-16le_bom/data-pass.yml b/testdata/utf-16le_bom/data-pass.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5f8246f7a287a74d96393d161c2db0bee264afb2
Binary files /dev/null and b/testdata/utf-16le_bom/data-pass.yml differ
diff --git a/testdata/utf-16le_bom/schema.json b/testdata/utf-16le_bom/schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..eb89b0044c97e48efad7b024b333420b67061031
Binary files /dev/null and b/testdata/utf-16le_bom/schema.json differ
diff --git a/testdata/utf-16le_bom/schema.yml b/testdata/utf-16le_bom/schema.yml
new file mode 100644
index 0000000000000000000000000000000000000000..43fc6b438ed49ca5f56cae6b22b6de8cd1bed4aa
Binary files /dev/null and b/testdata/utf-16le_bom/schema.yml differ
diff --git a/testdata/data-error.json b/testdata/utf-8/data-error.json
similarity index 100%
rename from testdata/data-error.json
rename to testdata/utf-8/data-error.json
diff --git a/testdata/data-error.yml b/testdata/utf-8/data-error.yml
similarity index 100%
rename from testdata/data-error.yml
rename to testdata/utf-8/data-error.yml
diff --git a/testdata/data-fail.json b/testdata/utf-8/data-fail.json
similarity index 100%
rename from testdata/data-fail.json
rename to testdata/utf-8/data-fail.json
diff --git a/testdata/data-fail.yml b/testdata/utf-8/data-fail.yml
similarity index 100%
rename from testdata/data-fail.yml
rename to testdata/utf-8/data-fail.yml
diff --git a/testdata/data-pass.json b/testdata/utf-8/data-pass.json
similarity index 100%
rename from testdata/data-pass.json
rename to testdata/utf-8/data-pass.json
diff --git a/testdata/data-pass.yml b/testdata/utf-8/data-pass.yml
similarity index 100%
rename from testdata/data-pass.yml
rename to testdata/utf-8/data-pass.yml
diff --git a/testdata/schema.json b/testdata/utf-8/schema.json
similarity index 100%
rename from testdata/schema.json
rename to testdata/utf-8/schema.json
diff --git a/testdata/schema.yml b/testdata/utf-8/schema.yml
similarity index 100%
rename from testdata/schema.yml
rename to testdata/utf-8/schema.yml
diff --git a/testdata/utf-8_bom/data-error.json b/testdata/utf-8_bom/data-error.json
new file mode 100644
index 0000000000000000000000000000000000000000..6e61837a202f8282c00d8b75d4cda12570e6e98b
--- /dev/null
+++ b/testdata/utf-8_bom/data-error.json
@@ -0,0 +1 @@
+not valid json
diff --git a/testdata/utf-8_bom/data-error.yml b/testdata/utf-8_bom/data-error.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f9b03bcfd72141d1afe93163a026e766fbadc6a6
--- /dev/null
+++ b/testdata/utf-8_bom/data-error.yml
@@ -0,0 +1 @@
+invalid: "an escaped \' single quote is not valid yaml
\ No newline at end of file
diff --git a/testdata/utf-8_bom/data-fail.json b/testdata/utf-8_bom/data-fail.json
new file mode 100644
index 0000000000000000000000000000000000000000..053cc03634136a99ba1e6daca33b05e6ac734444
--- /dev/null
+++ b/testdata/utf-8_bom/data-fail.json
@@ -0,0 +1,3 @@
+{
+ "bar": "missing foo"
+}
diff --git a/testdata/utf-8_bom/data-fail.yml b/testdata/utf-8_bom/data-fail.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b07635a4803565a2497391edc5c5ebff418e9c6f
--- /dev/null
+++ b/testdata/utf-8_bom/data-fail.yml
@@ -0,0 +1,2 @@
+---
+bar: missing foo
diff --git a/testdata/utf-8_bom/data-pass.json b/testdata/utf-8_bom/data-pass.json
new file mode 100644
index 0000000000000000000000000000000000000000..e091c96acf49f33c94fdde26bde9225366c6f8c3
--- /dev/null
+++ b/testdata/utf-8_bom/data-pass.json
@@ -0,0 +1,4 @@
+{
+ "foo": "asdf",
+ "bar": "zxcv"
+}
diff --git a/testdata/utf-8_bom/data-pass.yml b/testdata/utf-8_bom/data-pass.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d7e16a717021718dc7f868b195fb4a1c0f7482da
--- /dev/null
+++ b/testdata/utf-8_bom/data-pass.yml
@@ -0,0 +1,3 @@
+---
+foo: asdf
+bar: zxcv
diff --git a/testdata/utf-8_bom/schema.json b/testdata/utf-8_bom/schema.json
new file mode 100644
index 0000000000000000000000000000000000000000..5e26417b41302c16c78dc7199176d9d6b4f5a6bb
--- /dev/null
+++ b/testdata/utf-8_bom/schema.json
@@ -0,0 +1,7 @@
+{
+ "properties": {
+ "foo": { "type": "string" },
+ "bar": {}
+ },
+ "required": ["foo"]
+}
diff --git a/testdata/utf-8_bom/schema.yml b/testdata/utf-8_bom/schema.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f2c99d56aabc068942d2b9da22d69cb59600ffc7
--- /dev/null
+++ b/testdata/utf-8_bom/schema.yml
@@ -0,0 +1,7 @@
+---
+properties:
+ foo:
+ type: string
+ bar: {}
+required:
+ - foo