From 1dc00388a2797196db92b94091e2421d599fe60a Mon Sep 17 00:00:00 2001 From: kingthorin Date: Thu, 21 May 2020 08:28:21 -0400 Subject: [PATCH 1/9] Add GitHub Action to validate apps.json against schema.json Signed-off-by: kingthorin --- .github/workflows/schema-check.yml | 32 +++++++ schema.json | 135 +++++++++++++++++++---------- 2 files changed, 123 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/schema-check.yml diff --git a/.github/workflows/schema-check.yml b/.github/workflows/schema-check.yml new file mode 100644 index 000000000..ec10d3079 --- /dev/null +++ b/.github/workflows/schema-check.yml @@ -0,0 +1,32 @@ +name: JSON Schema Check + +on: + pull_request: + branches: + - master + paths: + - 'src/apps.json' + +jobs: + schema-check: + runs-on: ubuntu-latest + env: + CI: true + steps: + - name: Setup Action + uses: actions/checkout@v2 + - name: Setup Node + uses: actions/setup-node@v1 + with: + node-version: 13.x + - name: Install dependencies + run: npm install -g ajv-cli + - name: Run schema check + run: | + ajv validate -s schema.json -d src/apps.json +# - name: Upload list of broken links +# if: failure() && github.event_name == 'pull_request' +# uses: actions/upload-artifact@v1 +# with: +# name: broken-links +# path: brokenlinks.txt \ No newline at end of file diff --git a/schema.json b/schema.json index 45b52ac12..6ba762af5 100644 --- a/schema.json +++ b/schema.json @@ -2,18 +2,18 @@ "title": "Wappalyzer Schema", "type": "object", "additionalProperties": false, + "required": ["categories","apps"], "properties": { "$schema": { "type": "string" }, "categories": { "type": "object", - "required": true, + "minProperties": 64, "additionalProperties": false, "patternProperties": { "^[0-9]+$": { "type": "object", - "required": true, "properties": { "priority": { "type": "number" @@ -27,89 +27,136 @@ }, "apps": { "type": "object", - "required": true, "additionalProperties": { "additionalProperties": false, + "required": ["cats","website"], "properties": { "cats": { "type": "array", "items": { "type": "number" }, - "required": true + "minItems": 1 }, "cpe": { - "type": "string" + "type": "string", + "pattern": "^(?!\\s*$).+" }, "cookies": { "type": "object", - "additionalProperties": { - "type": "string" + "additionalProperties": false, + "patternProperties": { + "^.+$": { + }, + "additionalProperties": { + "type": "string" + } } }, "js": { "type": "object", - "additionalProperties": { - "type": "string" + "additionalProperties": false, + "patternProperties": { + "^.+$": { + }, + "additionalProperties": { + "type": "string" + } } }, "headers": { "type": "object", - "additionalProperties": { - "type": "string" + "additionalProperties": false, + "patternProperties": { + "^.+$": { + }, + "additionalProperties": { + "type": "string" + } } }, "html": { - "type": [ - "string", - "array" - ], - "items": { - "type": "string" - } + "oneOf": [ + { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + }, + { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + ] }, "excludes": { - "type": [ - "string", - "array" - ], - "items": { - "type": "string" - } + "oneOf": [ + { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + }, + { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + ] }, "implies": { - "type": [ - "string", - "array" - ], - "items": { - "type": "string" - } + "oneOf": [ + { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + }, + { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + ] }, "meta": { "type": "object", - "additionalProperties": { - "type": "string" + "additionalProperties": false, + "patternProperties": { + "^.+$": { + }, + "additionalProperties": { + "type": "string" + } } }, "script": { - "type": [ - "string", - "array" - ], - "items": { - "type": "string" - } + "oneOf": [ + { + "type": "array", + "items": { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + }, + { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + ] }, "url": { - "type": "string" + "type": "string", + "pattern": "^(?!\\s*$).+" }, "website": { "type": "string", - "required": true + "pattern": "^(?!\\s*$).+" }, "icon": { - "type": "string" + "type": "string", + "pattern": "^(?!\\s*$).+" } } } From 46865aa22ee9826aa877192281aca4dd68a7bedf Mon Sep 17 00:00:00 2001 From: kingthorin Date: Thu, 21 May 2020 08:32:50 -0400 Subject: [PATCH 2/9] Test --- src/apps.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/apps.json b/src/apps.json index 41f87971e..7d3f0472e 100644 --- a/src/apps.json +++ b/src/apps.json @@ -11961,6 +11961,9 @@ "cookie_name": "NivCMSPro" }, "icon": "nivsoft.png", + "js": { + "": "" + }, "headers": { "X-Powered-By": "NivCMS" }, From b6a003c97776ec141328ff19f46a154fcd32bfa8 Mon Sep 17 00:00:00 2001 From: kingthorin Date: Thu, 21 May 2020 08:35:23 -0400 Subject: [PATCH 3/9] Remove spurious property "env" on ClickFUnnels entry Signed-off-by: kingthorin --- src/apps.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/apps.json b/src/apps.json index 7d3f0472e..f1e05f66e 100644 --- a/src/apps.json +++ b/src/apps.json @@ -1608,7 +1608,6 @@ 32 ], "html": " Date: Thu, 21 May 2020 08:36:53 -0400 Subject: [PATCH 4/9] Cleanup action Signed-off-by: kingthorin --- .github/workflows/schema-check.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/schema-check.yml b/.github/workflows/schema-check.yml index ec10d3079..19da87a7c 100644 --- a/.github/workflows/schema-check.yml +++ b/.github/workflows/schema-check.yml @@ -24,9 +24,3 @@ jobs: - name: Run schema check run: | ajv validate -s schema.json -d src/apps.json -# - name: Upload list of broken links -# if: failure() && github.event_name == 'pull_request' -# uses: actions/upload-artifact@v1 -# with: -# name: broken-links -# path: brokenlinks.txt \ No newline at end of file From d7b83352427817ba1dd512b5acc6046966893756 Mon Sep 17 00:00:00 2001 From: kingthorin Date: Thu, 21 May 2020 08:39:10 -0400 Subject: [PATCH 5/9] Tabs not spaces in schema.json Signed-off-by: kingthorin --- schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema.json b/schema.json index 6ba762af5..428284fa1 100644 --- a/schema.json +++ b/schema.json @@ -9,7 +9,7 @@ }, "categories": { "type": "object", - "minProperties": 64, + "minProperties": 64, "additionalProperties": false, "patternProperties": { "^[0-9]+$": { From 3de0f31a9073fb908c62aa4419e888d4eb814075 Mon Sep 17 00:00:00 2001 From: kingthorin Date: Thu, 21 May 2020 09:04:09 -0400 Subject: [PATCH 6/9] Collect all errors & display additional context Signed-off-by: kingthorin --- .github/workflows/schema-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/schema-check.yml b/.github/workflows/schema-check.yml index 19da87a7c..5f2a0088d 100644 --- a/.github/workflows/schema-check.yml +++ b/.github/workflows/schema-check.yml @@ -23,4 +23,4 @@ jobs: run: npm install -g ajv-cli - name: Run schema check run: | - ajv validate -s schema.json -d src/apps.json + ajv validate -s schema.json -d src/apps.json --all-errors --error-data-path=property From 0e6555836ddf6912b9cb2ec4f2b7e1a4e06a57aa Mon Sep 17 00:00:00 2001 From: Kingthorin Date: Thu, 21 May 2020 19:33:14 -0400 Subject: [PATCH 7/9] Also run if schema is changed Signed-off-by: Kingthorin --- .github/workflows/schema-check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/schema-check.yml b/.github/workflows/schema-check.yml index 5f2a0088d..386550a20 100644 --- a/.github/workflows/schema-check.yml +++ b/.github/workflows/schema-check.yml @@ -6,6 +6,7 @@ on: - master paths: - 'src/apps.json' + - 'schema.json' jobs: schema-check: From 987703e68df247e1def775e3ba524534714a069f Mon Sep 17 00:00:00 2001 From: Kingthorin Date: Thu, 21 May 2020 19:50:53 -0400 Subject: [PATCH 8/9] Simplify schema.json using definitions/refs Signed-off-by: Kingthorin --- schema.json | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/schema.json b/schema.json index 428284fa1..c5a875676 100644 --- a/schema.json +++ b/schema.json @@ -1,5 +1,11 @@ { "title": "Wappalyzer Schema", + "definitions": { + "non-empty-non-blank-sting": { + "type": "string", + "pattern": "^(?!\\s*$).+" + } + }, "type": "object", "additionalProperties": false, "required": ["categories","apps"], @@ -39,8 +45,7 @@ "minItems": 1 }, "cpe": { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" }, "cookies": { "type": "object", @@ -80,13 +85,11 @@ { "type": "array", "items": { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } }, { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } ] }, @@ -95,13 +98,11 @@ { "type": "array", "items": { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } }, { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } ] }, @@ -110,13 +111,11 @@ { "type": "array", "items": { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } }, { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } ] }, @@ -136,27 +135,22 @@ { "type": "array", "items": { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } }, { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } ] }, "url": { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" }, "website": { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" }, "icon": { - "type": "string", - "pattern": "^(?!\\s*$).+" + "$ref": "#/definitions/non-empty-non-blank-sting" } } } From b40433f97a242c42df8dba39392d69a422341ea9 Mon Sep 17 00:00:00 2001 From: Kingthorin Date: Thu, 21 May 2020 20:00:23 -0400 Subject: [PATCH 9/9] Remove workflow, leaving schema.json and apps.json Signed-off-by: Kingthorin --- .github/workflows/schema-check.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/workflows/schema-check.yml diff --git a/.github/workflows/schema-check.yml b/.github/workflows/schema-check.yml deleted file mode 100644 index 386550a20..000000000 --- a/.github/workflows/schema-check.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: JSON Schema Check - -on: - pull_request: - branches: - - master - paths: - - 'src/apps.json' - - 'schema.json' - -jobs: - schema-check: - runs-on: ubuntu-latest - env: - CI: true - steps: - - name: Setup Action - uses: actions/checkout@v2 - - name: Setup Node - uses: actions/setup-node@v1 - with: - node-version: 13.x - - name: Install dependencies - run: npm install -g ajv-cli - - name: Run schema check - run: | - ajv validate -s schema.json -d src/apps.json --all-errors --error-data-path=property