From 3a0fb3d9f7919a141546502e8c900a283536b368 Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Thu, 12 Jun 2025 17:07:44 -0700 Subject: [PATCH 1/9] Use full schema (schema-base.yaml) for coverage But don't count standard JSON Schema keywords. --- scripts/schema-test-coverage.mjs | 74 ++++++++++++++++++++++++++++++-- scripts/schema-test-coverage.sh | 2 +- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 0b2050ea60..d3a2ea3466 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -1,10 +1,11 @@ +import { readFileSync } from "node:fs"; import { readdir, readFile } from "node:fs/promises"; import YAML from "yaml"; import { join } from "node:path"; import { argv } from "node:process"; -import { validate } from "@hyperjump/json-schema/draft-2020-12"; +import { registerSchema, validate } from "@hyperjump/json-schema/draft-2020-12"; import "@hyperjump/json-schema/draft-04"; -import { BASIC } from "@hyperjump/json-schema/experimental"; +import { BASIC, addKeyword, defineVocabulary } from "@hyperjump/json-schema/experimental"; /** * @import { EvaluationPlugin } from "@hyperjump/json-schema/experimental" @@ -45,7 +46,10 @@ class TestCoveragePlugin { this.allLocations = []; for (const schemaLocation in context.ast) { - if (schemaLocation === "metaData") { + if ( + schemaLocation === "metaData" || + schemaLocation.includes("json-schema.org") + ) { continue; } @@ -110,6 +114,68 @@ const runTests = async (schemaUri, testDirectory) => { }; }; +addKeyword({ + id: "https://spec.openapis.org/oas/schema/vocab/keyword/discriminator", + interpret: (discriminator, instance, context) => { + return true; + }, + /* discriminator is not exactly an annotation, but it's not allowed + * to change the validation outcome (hence returing true from interopret()) + * and for our purposes of testing, this is sufficient. + */ + annotation: (discriminator) => { + return discriminator; + }, +}); + +addKeyword({ + id: "https://spec.openapis.org/oas/schema/vocab/keyword/example", + interpret: (example, instance, context) => { + return true; + }, + annotation: (example) => { + return example; + }, +}); + +addKeyword({ + id: "https://spec.openapis.org/oas/schema/vocab/keyword/externalDocs", + interpret: (externalDocs, instance, context) => { + return true; + }, + annotation: (externalDocs) => { + return externalDocs; + }, +}); + +addKeyword({ + id: "https://spec.openapis.org/oas/schema/vocab/keyword/xml", + interpret: (xml, instance, context) => { + return true; + }, + annotation: (xml) => { + return xml; + }, +}); + +defineVocabulary( + "https://spec.openapis.org/oas/3.1/vocab/base", + { + "discriminator": "https://spec.openapis.org/oas/schema/vocab/keyword/discriminator", + "example": "https://spec.openapis.org/oas/schema/vocab/keyword/example", + "externalDocs": "https://spec.openapis.org/oas/schema/vocab/keyword/externalDocs", + "xml": "https://spec.openapis.org/oas/schema/vocab/keyword/xml", + }, +); + +const parseYamlFromFile = (filePath) => { + const schemaYaml = readFileSync(filePath, "utf8"); + return YAML.parse(schemaYaml, { prettyErrors: true }); +}; +registerSchema(parseYamlFromFile("./src/schemas/validation/meta.yaml")); +registerSchema(parseYamlFromFile("./src/schemas/validation/dialect.yaml")); +registerSchema(parseYamlFromFile("./src/schemas/validation/schema.yaml")); + /////////////////////////////////////////////////////////////////////////////// const { allLocations, visitedLocations } = await runTests(argv[2], argv[3]); @@ -134,4 +200,4 @@ console.log( if (visitedLocations.size != allLocations.length) { process.exitCode = 1; -} \ No newline at end of file +} diff --git a/scripts/schema-test-coverage.sh b/scripts/schema-test-coverage.sh index 825a254e26..f00b661b0b 100755 --- a/scripts/schema-test-coverage.sh +++ b/scripts/schema-test-coverage.sh @@ -12,7 +12,7 @@ echo echo "Schema Test Coverage" echo -node scripts/schema-test-coverage.mjs src/schemas/validation/schema.yaml tests/schema/pass +node scripts/schema-test-coverage.mjs src/schemas/validation/schema-base.yaml tests/schema/pass rc=$? [[ "$branch" == "dev" ]] || exit $rc From 0fadf2e67f0f7f325017302158bcca66220e94e4 Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Thu, 12 Jun 2025 19:22:01 -0700 Subject: [PATCH 2/9] Do not expect coverage of unreachable schema There needs to be a local `$dynamicAnchor`, but it is never actually evaluated. --- scripts/schema-test-coverage.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index d3a2ea3466..960c9bb328 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -48,7 +48,12 @@ class TestCoveragePlugin { for (const schemaLocation in context.ast) { if ( schemaLocation === "metaData" || - schemaLocation.includes("json-schema.org") + // Do not reqiure coverage of standard JSON Schema + schemaLocation.includes("json-schema.org") || + // Do not require coverage of default $dynamicAnchor + // schemas, as they are not expected to be reached + // schemaLocation.includes("/schema/WORK-IN-PROGRESS#/$defs/schema/") + schemaLocation.endsWith("/schema/WORK-IN-PROGRESS#/$defs/schema") ) { continue; } From f66d94ca0697ecbaddf3eff006562cdbbf031ad9 Mon Sep 17 00:00:00 2001 From: Henry Andrews Date: Fri, 13 Jun 2025 07:43:58 -0700 Subject: [PATCH 3/9] Fix spelling Co-authored-by: Ralf Handl --- scripts/schema-test-coverage.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 960c9bb328..51f5b1ba05 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -48,7 +48,7 @@ class TestCoveragePlugin { for (const schemaLocation in context.ast) { if ( schemaLocation === "metaData" || - // Do not reqiure coverage of standard JSON Schema + // Do not require coverage of standard JSON Schema schemaLocation.includes("json-schema.org") || // Do not require coverage of default $dynamicAnchor // schemas, as they are not expected to be reached From d986f8864acc3fbc06d3a0c6956ab0298359b368 Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Fri, 13 Jun 2025 07:55:21 -0700 Subject: [PATCH 4/9] Fix coverage calculation Co-authored-by: Ralf Handl --- scripts/schema-test-coverage.mjs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 51f5b1ba05..2dee442f1e 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -193,16 +193,13 @@ if (notCovered.length > 0) { const firstNotCovered = notCovered.slice(0, maxNotCovered); if (notCovered.length > maxNotCovered) firstNotCovered.push("..."); console.log(firstNotCovered); + process.exitCode = 1; } console.log( "Covered:", - visitedLocations.size, + (allocations.length - notCovered.length), "of", allLocations.length, "(" + Math.floor((visitedLocations.size / allLocations.length) * 100) + "%)", ); - -if (visitedLocations.size != allLocations.length) { - process.exitCode = 1; -} From 20eec3a233821bad82eedca71b304cb701052ec9 Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Fri, 13 Jun 2025 07:59:06 -0700 Subject: [PATCH 5/9] Remove stray commented-out line. --- scripts/schema-test-coverage.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 2dee442f1e..0fbfd0f8e8 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -52,7 +52,6 @@ class TestCoveragePlugin { schemaLocation.includes("json-schema.org") || // Do not require coverage of default $dynamicAnchor // schemas, as they are not expected to be reached - // schemaLocation.includes("/schema/WORK-IN-PROGRESS#/$defs/schema/") schemaLocation.endsWith("/schema/WORK-IN-PROGRESS#/$defs/schema") ) { continue; From 8136caab5fe1aed5cef961a42d2a19361f32f80c Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Fri, 13 Jun 2025 08:14:35 -0700 Subject: [PATCH 6/9] Use full schema-base to run schema tests --- tests/schema/schema.test.mjs | 63 ++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/tests/schema/schema.test.mjs b/tests/schema/schema.test.mjs index 362ccc856c..42f96813c7 100644 --- a/tests/schema/schema.test.mjs +++ b/tests/schema/schema.test.mjs @@ -1,7 +1,7 @@ import { readdirSync, readFileSync } from "node:fs"; import YAML from "yaml"; -import { validate, setMetaSchemaOutputFormat } from "@hyperjump/json-schema/openapi-3-1"; -import { BASIC } from "@hyperjump/json-schema/experimental"; +import { registerSchema, validate, setMetaSchemaOutputFormat } from "@hyperjump/json-schema/openapi-3-1"; +import { BASIC, addKeyword, defineVocabulary } from "@hyperjump/json-schema/experimental"; import { describe, test, expect } from "vitest"; import contentTypeParser from "content-type"; @@ -26,7 +26,64 @@ const parseYamlFromFile = (filePath) => { setMetaSchemaOutputFormat(BASIC); -const validateOpenApi = await validate("./src/schemas/validation/schema.yaml"); +addKeyword({ + id: "https://spec.openapis.org/oas/schema/vocab/keyword/discriminator", + interpret: (discriminator, instance, context) => { + return true; + }, + /* discriminator is not exactly an annotation, but it's not allowed + * to change the validation outcome (hence returing true from interopret()) + * and for our purposes of testing, this is sufficient. + */ + annotation: (discriminator) => { + return discriminator; + }, +}); + +addKeyword({ + id: "https://spec.openapis.org/oas/schema/vocab/keyword/example", + interpret: (example, instance, context) => { + return true; + }, + annotation: (example) => { + return example; + }, +}); + +addKeyword({ + id: "https://spec.openapis.org/oas/schema/vocab/keyword/externalDocs", + interpret: (externalDocs, instance, context) => { + return true; + }, + annotation: (externalDocs) => { + return externalDocs; + }, +}); + +addKeyword({ + id: "https://spec.openapis.org/oas/schema/vocab/keyword/xml", + interpret: (xml, instance, context) => { + return true; + }, + annotation: (xml) => { + return xml; + }, +}); + +defineVocabulary( + "https://spec.openapis.org/oas/3.1/vocab/base", + { + "discriminator": "https://spec.openapis.org/oas/schema/vocab/keyword/discriminator", + "example": "https://spec.openapis.org/oas/schema/vocab/keyword/example", + "externalDocs": "https://spec.openapis.org/oas/schema/vocab/keyword/externalDocs", + "xml": "https://spec.openapis.org/oas/schema/vocab/keyword/xml", + }, +); + +registerSchema(parseYamlFromFile("./src/schemas/validation/meta.yaml")); +registerSchema(parseYamlFromFile("./src/schemas/validation/dialect.yaml")); +registerSchema(parseYamlFromFile("./src/schemas/validation/schema.yaml")); +const validateOpenApi = await validate("./src/schemas/validation/schema-base.yaml"); const fixtures = './tests/schema'; describe("v3.1", () => { From 3a0a0258f4cadf2a70a9373c69dff049d1968783 Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Thu, 12 Jun 2025 16:01:14 -0700 Subject: [PATCH 7/9] Use matching jsonSchemaDialect Since we are testing with a placeholder, we need to match the placeholder. This will unfortunately need to be different on each new release line branch, so let's separate this test case into its own file. --- tests/schema/pass/json_schema_dialect.yaml | 15 +++++++++++++++ tests/schema/pass/mega.yaml | 1 - 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/schema/pass/json_schema_dialect.yaml diff --git a/tests/schema/pass/json_schema_dialect.yaml b/tests/schema/pass/json_schema_dialect.yaml new file mode 100644 index 0000000000..ae0ed863b3 --- /dev/null +++ b/tests/schema/pass/json_schema_dialect.yaml @@ -0,0 +1,15 @@ +openapi: 3.1.0 +info: + summary: Testing jsonSchemaDialect + title: My API + version: 1.0.0 + license: + name: Apache 2.0 + identifier: Apache-2.0 +jsonSchemaDialect: https://spec.openapis.org/oas/3.1/dialect/WORK-IN-PROGRESS +components: + schemas: + WithDollarSchema: + $id: "locked-metaschema" + $schema: https://spec.openapis.org/oas/3.1/dialect/WORK-IN-PROGRESS +paths: {} diff --git a/tests/schema/pass/mega.yaml b/tests/schema/pass/mega.yaml index 8838c03a6d..98ce577dce 100644 --- a/tests/schema/pass/mega.yaml +++ b/tests/schema/pass/mega.yaml @@ -6,7 +6,6 @@ info: license: name: Apache 2.0 identifier: Apache-2.0 -jsonSchemaDialect: https://spec.openapis.org/oas/3.1/dialect/base paths: /: get: From 0c75419861f140ff4ba95d5c0180dd355eb13858 Mon Sep 17 00:00:00 2001 From: Henry Andrews Date: Fri, 13 Jun 2025 08:27:08 -0700 Subject: [PATCH 8/9] Update scripts/schema-test-coverage.mjs Co-authored-by: Ralf Handl --- scripts/schema-test-coverage.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 0fbfd0f8e8..3eea24b060 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -200,5 +200,5 @@ console.log( (allocations.length - notCovered.length), "of", allLocations.length, - "(" + Math.floor((visitedLocations.size / allLocations.length) * 100) + "%)", + "(" + Math.floor(((allocations.length - notCovered.length) / allLocations.length) * 100) + "%)", ); From 3b6551c14253e6a99301ccc9d0f91307806295ca Mon Sep 17 00:00:00 2001 From: "Henry H. Andrews" Date: Fri, 13 Jun 2025 08:48:45 -0700 Subject: [PATCH 9/9] Fix typos --- scripts/schema-test-coverage.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 3eea24b060..82e7f822b2 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -197,8 +197,8 @@ if (notCovered.length > 0) { console.log( "Covered:", - (allocations.length - notCovered.length), + (allLocations.length - notCovered.length), "of", allLocations.length, - "(" + Math.floor(((allocations.length - notCovered.length) / allLocations.length) * 100) + "%)", + "(" + Math.floor(((allLocations.length - notCovered.length) / allLocations.length) * 100) + "%)", );