diff --git a/encoding/jsonschema/external_test.go b/encoding/jsonschema/external_test.go index aafbfcb9f..6ea3c3402 100644 --- a/encoding/jsonschema/external_test.go +++ b/encoding/jsonschema/external_test.go @@ -40,6 +40,7 @@ import ( // The commit below references the JSON schema test main branch as of Sun May 19 19:01:03 2024 +0300 //go:generate go run vendor_external.go -- 9fc880bfb6d8ccd093bc82431f17d13681ffae8e +//go:generate go run teststats.go -o external_teststats.txt const testDir = "testdata/external" diff --git a/encoding/jsonschema/external_teststats.txt b/encoding/jsonschema/external_teststats.txt new file mode 100644 index 000000000..4bccdc3a6 --- /dev/null +++ b/encoding/jsonschema/external_teststats.txt @@ -0,0 +1,10 @@ +# Generated by teststats. DO NOT EDIT +v2: + schema extract (pass / total): 975 / 1637 = 59.6% + tests (pass / total): 3140 / 7175 = 43.8% + tests on extracted schemas (pass / total): 3140 / 3546 = 88.6% + +v3: + schema extract (pass / total): 967 / 1637 = 59.1% + tests (pass / total): 3074 / 7175 = 42.8% + tests on extracted schemas (pass / total): 3074 / 3538 = 86.9% diff --git a/encoding/jsonschema/teststats.go b/encoding/jsonschema/teststats.go index b33609640..33e487195 100644 --- a/encoding/jsonschema/teststats.go +++ b/encoding/jsonschema/teststats.go @@ -20,7 +20,9 @@ package main import ( "flag" "fmt" + "io" "log" + "os" "path" "sort" @@ -28,28 +30,40 @@ import ( "cuelang.org/go/encoding/jsonschema/internal/externaltest" ) -var list = flag.String("list", "", "list all failed tests for a given evaluator version") +var ( + list = flag.String("list", "", "list all failed tests for a given evaluator version") + out = flag.String("o", "", "write output to a file") +) const testDir = "testdata/external" func main() { + flag.Parse() tests, err := externaltest.ReadTestDir(testDir) if err != nil { log.Fatal(err) } - flag.Parse() + outw := os.Stdout + if *out != "" { + var err error + outw, err = os.Create(*out) + if err != nil { + log.Fatal(err) + } + fmt.Fprintf(outw, "# Generated by teststats. DO NOT EDIT\n") + } if *list != "" { - listFailures(*list, tests) + listFailures(outw, *list, tests) } else { - fmt.Printf("v2:\n") - showStats("v2", tests) - fmt.Println() - fmt.Printf("v3:\n") - showStats("v3", tests) + fmt.Fprintf(outw, "v2:\n") + showStats(outw, "v2", tests) + fmt.Fprintf(outw, "\n") + fmt.Fprintf(outw, "v3:\n") + showStats(outw, "v3", tests) } } -func showStats(version string, tests map[string][]*externaltest.Schema) { +func showStats(outw io.Writer, version string, tests map[string][]*externaltest.Schema) { schemaOK := 0 schemaTot := 0 testOK := 0 @@ -76,17 +90,17 @@ func showStats(version string, tests map[string][]*externaltest.Schema) { } } } - fmt.Printf("\tschema extract (pass / total): %d / %d = %.1f%%\n", schemaOK, schemaTot, percent(schemaOK, schemaTot)) - fmt.Printf("\ttests (pass / total): %d / %d = %.1f%%\n", testOK, testTot, percent(testOK, testTot)) - fmt.Printf("\ttests on extracted schemas (pass / total): %d / %d = %.1f%%\n", schemaOKTestOK, schemaOKTestTot, percent(schemaOKTestOK, schemaOKTestTot)) + fmt.Fprintf(outw, "\tschema extract (pass / total): %d / %d = %.1f%%\n", schemaOK, schemaTot, percent(schemaOK, schemaTot)) + fmt.Fprintf(outw, "\ttests (pass / total): %d / %d = %.1f%%\n", testOK, testTot, percent(testOK, testTot)) + fmt.Fprintf(outw, "\ttests on extracted schemas (pass / total): %d / %d = %.1f%%\n", schemaOKTestOK, schemaOKTestTot, percent(schemaOKTestOK, schemaOKTestTot)) } -func listFailures(version string, tests map[string][]*externaltest.Schema) { +func listFailures(outw io.Writer, version string, tests map[string][]*externaltest.Schema) { for _, filename := range sortedKeys(tests) { schemas := tests[filename] for _, schema := range schemas { if schema.Skip[version] != "" { - fmt.Printf("%s: schema fail (%s)\n", testdataPos(schema), schema.Description) + fmt.Fprintf(outw, "%s: schema fail (%s)\n", testdataPos(schema), schema.Description) continue } for _, test := range schema.Tests { @@ -95,7 +109,7 @@ func listFailures(version string, tests map[string][]*externaltest.Schema) { if !test.Valid { reason = "unexpected success" } - fmt.Printf("%s: %s (%s; %s)\n", testdataPos(test), reason, schema.Description, test.Description) + fmt.Fprintf(outw, "%s: %s (%s; %s)\n", testdataPos(test), reason, schema.Description, test.Description) } } }