encoding/jsonschema: encode oneOf etc using matchN

This CL updates JSON Schema generation to use the newly added `matchN` primitive for `allOf`, `anyOf` and `oneOf`. This has a much closer correlation with the JSON Schema primitives than the current approach of using CUE's `&` and `|` operators. Specifically: - the schema arguments should not affect the final result other than to validate it, but both `&` and `|` can affect the result. - the result could become non-concrete due to `|` ambiguity in `anyOf`. Although this does fix a bunch of issues, there are some regressions. See issues #3418, #3420, #3422 for details. Comparative test stats (pre-matchN before post-matchN): ``` v2: schema extract (pass / total): 971 / 1637 = 59.3% schema extract (pass / total): 975 / 1637 = 59.6% tests (pass / total): 3081 / 7175 = 42.9% tests (pass / total): 3140 / 7175 = 43.8% tests on extracted schemas (pass / total): 3081 / 3542 = 87.0% tests on extracted schemas (pass / total): 3140 / 3546 = 88.6% v3: schema extract (pass / total): 971 / 1637 = 59.3% schema extract (pass / total): 967 / 1637 = 59.1% tests (pass / total): 3063 / 7175 = 42.7% tests (pass / total): 3074 / 7175 = 42.8% tests on extracted schemas (pass / total): 3063 / 3542 = 86.5% tests on extracted schemas (pass / total): 3074 / 3538 = 86.9% ``` This change also requires that we update the CI generated code and remove the workaround for the previous `oneOf` limitation. For #3380 For #3165 Signed-off-by: Roger Peppe <rogpeppe@gmail.com> Change-Id: I2630a6d2b1614b2479802e788c16249d2cf4aa6b Dispatch-Trailer: {"type":"trybot","CL":1200526,"patchset":3,"ref":"refs/changes/26/1200526/3","targetBranch":"master"}
cue-lang · Sep 3, 2024 · 7762d5e · 7762d5e
1 parent 9cf4ee4
commit 7762d5e
Show file tree

Hide file tree

Showing 38 changed files with 472 additions and 432 deletions.
diff --git a/.github/workflows/evict_caches.yml b/.github/workflows/evict_caches.yml
@@ -21,8 +21,8 @@ jobs:
         run: touch -t 202211302355 $(find * -type d)
       - name: Restore git file modification times
         uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe
-      - id: DispatchTrailer
-        name: Try to extract Dispatch-Trailer
+      - name: Try to extract Dispatch-Trailer
+        id: DispatchTrailer
         run: |-
           x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')"
           if [[ "$x" == "" ]]
@@ -38,11 +38,11 @@ jobs:
           echo "value<<EOD" >> $GITHUB_OUTPUT
           echo "$x" >> $GITHUB_OUTPUT
           echo "EOD" >> $GITHUB_OUTPUT
-      - if: |-
+      - name: Check we don't have Dispatch-Trailer on a protected branch
+        if: |-
           ((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"'))
-        name: Check we don't have Dispatch-Trailer on a protected branch
         run: |-
           echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch"
           false

diff --git a/.github/workflows/push_tip_to_trybot.yml b/.github/workflows/push_tip_to_trybot.yml
@@ -10,10 +10,6 @@ concurrency: push_tip_to_trybot
 jobs:
   push:
     runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-    if: ${{github.repository == 'cue-lang/cue'}}
     steps:
       - name: Write netrc file for cueckoo Gerrithub
         run: |-
@@ -49,3 +45,7 @@ jobs:
           	 echo "Giving up"
           	 exit 1
           fi
+    defaults:
+      run:
+        shell: bash
+    if: ${{github.repository == 'cue-lang/cue'}}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -14,10 +14,10 @@ concurrency: release
 jobs:
   goreleaser:
     runs-on: ubuntu-22.04
+    if: ${{github.repository == 'cue-lang/cue'}}
     defaults:
       run:
         shell: bash
-    if: ${{github.repository == 'cue-lang/cue'}}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -28,8 +28,8 @@ jobs:
         run: touch -t 202211302355 $(find * -type d)
       - name: Restore git file modification times
         uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe
-      - id: DispatchTrailer
-        name: Try to extract Dispatch-Trailer
+      - name: Try to extract Dispatch-Trailer
+        id: DispatchTrailer
         run: |-
           x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')"
           if [[ "$x" == "" ]]
@@ -45,11 +45,11 @@ jobs:
           echo "value<<EOD" >> $GITHUB_OUTPUT
           echo "$x" >> $GITHUB_OUTPUT
           echo "EOD" >> $GITHUB_OUTPUT
-      - if: |-
+      - name: Check we don't have Dispatch-Trailer on a protected branch
+        if: |-
           ((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"'))
-        name: Check we don't have Dispatch-Trailer on a protected branch
         run: |-
           echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch"
           false
@@ -76,13 +76,13 @@ jobs:
           install-only: true
           version: v2.2.0
       - name: Run GoReleaser with CUE
-        run: cue cmd release
-        working-directory: ./internal/ci/goreleaser
         env:
           GITHUB_TOKEN: ${{ secrets.CUECKOO_GITHUB_PAT }}
-      - if: startsWith(github.ref, 'refs/tags/v')
-        name: Re-test cuelang.org
+        run: cue cmd release
+        working-directory: ./internal/ci/goreleaser
+      - name: Re-test cuelang.org
+        if: startsWith(github.ref, 'refs/tags/v')
         run: 'curl -s -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.CUECKOO_GITHUB_PAT }}" -H "X-GitHub-Api-Version: 2022-11-28" --fail --request POST --data-binary "{\"event_type\":\"Re-test post release of ${GITHUB_REF##refs/tags/}\"}" https://api.github.com/repos/cue-lang/cuelang.org/dispatches'
-      - if: startsWith(github.ref, 'refs/tags/v')
-        name: Trigger unity build
+      - name: Trigger unity build
+        if: startsWith(github.ref, 'refs/tags/v')
         run: 'curl -s -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.PORCUEPINE_GITHUB_PAT }}" -H "X-GitHub-Api-Version: 2022-11-28" --fail --request POST --data-binary "{\"event_type\":\"Check against CUE ${GITHUB_REF##refs/tags/}\",\"client_payload\":{\"type\":\"unity\",\"payload\":{\"versions\":\"\\\"${GITHUB_REF##refs/tags/}\\\"\"}}}" https://api.github.com/repos/cue-unity/unity-private/dispatches'
diff --git a/.github/workflows/tip_triggers.yml b/.github/workflows/tip_triggers.yml
@@ -8,10 +8,10 @@ name: Triggers on push to tip
 jobs:
   push:
     runs-on: ubuntu-22.04
+    if: ${{github.repository == 'cue-lang/cue'}}
     defaults:
       run:
         shell: bash
-    if: ${{github.repository == 'cue-lang/cue'}}
     steps:
       - name: Trigger unity build
         run: 'curl -s -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.PORCUEPINE_GITHUB_PAT }}" -H "X-GitHub-Api-Version: 2022-11-28" --fail --request POST --data-binary "{\"event_type\":\"Check against ${GITHUB_SHA}\",\"client_payload\":{\"type\":\"unity\",\"payload\":{\"versions\":\"\\\"commit:${GITHUB_SHA}\\\"\"}}}" https://api.github.com/repos/cue-unity/unity-private/dispatches'
diff --git a/.github/workflows/trybot.yml b/.github/workflows/trybot.yml
@@ -9,8 +9,8 @@ name: TryBot
       - release-branch.*
     tags-ignore:
       - v*
-  pull_request: {}
   workflow_dispatch: {}
+  pull_request: {}
 jobs:
   test:
     strategy:
@@ -24,13 +24,13 @@ jobs:
           - macos-14
           - windows-2022
     runs-on: ${{ matrix.runner }}
-    defaults:
-      run:
-        shell: bash
     if: |-
       (contains(github.event.head_commit.message, '
       Dispatch-Trailer: {"type":"trybot"')) || ! (contains(github.event.head_commit.message, '
       Dispatch-Trailer: {"type":"'))
+    defaults:
+      run:
+        shell: bash
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -41,8 +41,8 @@ jobs:
         run: touch -t 202211302355 $(find * -type d)
       - name: Restore git file modification times
         uses: chetan/git-restore-mtime-action@075f9bc9d159805603419d50f794bd9f33252ebe
-      - id: DispatchTrailer
-        name: Try to extract Dispatch-Trailer
+      - name: Try to extract Dispatch-Trailer
+        id: DispatchTrailer
         run: |-
           x="$(git log -1 --pretty='%(trailers:key=Dispatch-Trailer,valueonly)')"
           if [[ "$x" == "" ]]
@@ -58,11 +58,11 @@ jobs:
           echo "value<<EOD" >> $GITHUB_OUTPUT
           echo "$x" >> $GITHUB_OUTPUT
           echo "EOD" >> $GITHUB_OUTPUT
-      - if: |-
+      - name: Check we don't have Dispatch-Trailer on a protected branch
+        if: |-
           ((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"')))) && (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"'))
-        name: Check we don't have Dispatch-Trailer on a protected branch
         run: |-
           echo "github.event.head_commit.message contains Dispatch-Trailer but we are on a protected branch"
           false
@@ -71,73 +71,73 @@ jobs:
         with:
           cache: false
           go-version: ${{ matrix.go-version }}
-      - id: go-mod-cache-dir
-        name: Get go mod cache directory
+      - name: Get go mod cache directory
+        id: go-mod-cache-dir
         run: echo "dir=$(go env GOMODCACHE)" >> ${GITHUB_OUTPUT}
-      - id: go-cache-dir
-        name: Get go build/test cache directory
+      - name: Get go build/test cache directory
+        id: go-cache-dir
         run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT}
-      - if: |-
-          (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
-          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
-        uses: actions/cache@v4
-        with:
+      - with:
           path: |-
             ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
             ${{ steps.go-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-${{ matrix.go-version }}-${{ github.run_id }}
           restore-keys: ${{ runner.os }}-${{ matrix.go-version }}
-      - if: |-
-          ! (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
+        if: |-
+          (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
-        uses: actions/cache/restore@v4
-        with:
+        uses: actions/cache@v4
+      - with:
           path: |-
             ${{ steps.go-mod-cache-dir.outputs.dir }}/cache/download
             ${{ steps.go-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-${{ matrix.go-version }}-${{ github.run_id }}
           restore-keys: ${{ runner.os }}-${{ matrix.go-version }}
+        uses: actions/cache/restore@v4
+        if: |-
+          ! (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
       - if: |-
           github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"')))) || github.ref == 'refs/heads/ci/test')
         run: go clean -testcache
-      - if: (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
-        name: Early git and code sanity checks
+      - name: Early git and code sanity checks
+        if: (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
         run: go run ./internal/ci/checks
-      - if: (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
-        name: Generate
+      - name: Generate
         run: go generate ./...
-      - if: |-
+        if: (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
+      - name: Test
+        if: |-
           ((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"')))) || !(matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
-        name: Test
         run: go test ./...
-      - if: (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
-        name: Test with -race
-        run: go test -race ./...
+      - name: Test with -race
         env:
           GORACE: atexit_sleep_ms=10
+        if: (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
+        run: go test -race ./...
       - name: Test with -tags=cuewasm
         run: go test -tags cuewasm ./cmd/cue/cmd ./cue/interpreter/wasm
       - name: gcloud auth for end-to-end tests
         id: auth
+        uses: google-github-actions/auth@v2
         if: |-
           github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
-        uses: google-github-actions/auth@v2
         with:
           credentials_json: ${{ secrets.E2E_GCLOUD_KEY }}
-      - if: |-
+      - name: gcloud setup for end-to-end tests
+        if: |-
           github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
           Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
-        name: gcloud setup for end-to-end tests
         uses: google-github-actions/setup-gcloud@v2
-      - if: |-
-          github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
-          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
-        name: End-to-end test
+      - name: End-to-end test
         env:
           CUE_TEST_LOGINS: ${{ secrets.E2E_CUE_LOGINS }}
+        if: |-
+          github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
+          Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.23.x' && matrix.runner == 'ubuntu-22.04')
         run: |-
           cd internal/_e2e
           go test -race
@@ -147,6 +147,6 @@ jobs:
           go vet ./...
           go mod tidy
           (cd internal/_e2e && go test -run=-)
-      - if: always()
-        name: Check that git is clean at the end of the job
+      - name: Check that git is clean at the end of the job
+        if: always()
         run: test -z "$(git status --porcelain)" || (git status; git diff; false)
diff --git a/.github/workflows/trybot_dispatch.yml b/.github/workflows/trybot_dispatch.yml
@@ -9,10 +9,10 @@ name: Dispatch trybot
 jobs:
   trybot:
     runs-on: ubuntu-22.04
+    if: ${{ ((github.ref == 'refs/heads/ci/test') && false) || github.event.client_payload.type == 'trybot' }}
     defaults:
       run:
         shell: bash
-    if: ${{ ((github.ref == 'refs/heads/ci/test') && false) || github.event.client_payload.type == 'trybot' }}
     steps:
       - name: Write netrc file for cueckoo Gerrithub
         run: |-
@@ -22,17 +22,17 @@ jobs:
           password ${{ secrets.CUECKOO_GERRITHUB_PASSWORD }}
           EOD
           chmod 600 ~/.netrc
-      - id: payload
+      - name: Write fake payload
+        id: payload
         if: github.repository == 'cue-lang/cue' && (github.ref == 'refs/heads/ci/test')
-        name: Write fake payload
         run: |-
           cat <<EOD >> $GITHUB_OUTPUT
           value<<DOE
           null
           DOE
           EOD
-      - if: github.event.client_payload.type != 'trybot'
-        name: Trigger TryBot (fake data)
+      - name: Trigger TryBot (fake data)
+        if: github.event.client_payload.type != 'trybot'
         run: |-
           mkdir tmpgit
           cd tmpgit
@@ -81,8 +81,8 @@ jobs:
           	echo "Giving up"
           	exit 1
           fi
-      - if: github.event.client_payload.type == 'trybot'
-        name: Trigger TryBot (repository_dispatch payload)
+      - name: Trigger TryBot (repository_dispatch payload)
+        if: github.event.client_payload.type == 'trybot'
         run: |-
           mkdir tmpgit
           cd tmpgit