diff --git a/.github/SGID_INDEX_ISSUE_TEMPLATE.md b/.github/SGID_INDEX_ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..b1b1900de5 --- /dev/null +++ b/.github/SGID_INDEX_ISSUE_TEMPLATE.md @@ -0,0 +1,8 @@ +--- +title: SGID Index Validation Issues +labels: sgid index validation +--- + +Validation errors have been detected in the [SGID Index](https://docs.google.com/spreadsheets/d/11ASS7LnxgpnD0jN4utzklREgMf1pcvYjcXcIcESHweQ/edit#gid=1024261148). This issue will be used to track the resolution of these issues. + +GitHub Actions should post a comment with the details shortly... diff --git a/.github/workflows/schedule.sgid-index-validation.yml b/.github/workflows/schedule.sgid-index-validation.yml new file mode 100644 index 0000000000..98f3ebd2ca --- /dev/null +++ b/.github/workflows/schedule.sgid-index-validation.yml @@ -0,0 +1,125 @@ +name: Schedule (SGID Index Validation) + +on: + schedule: + - cron: '0 0 * * 1-5' + workflow_dispatch: + issue_comment: + types: [created, edited] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: + issues: write + +jobs: + validate: + name: Validate SGID Index + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./src/scripts + env: + GOOGLE_PRIVATE_KEY: ${{ secrets.SA }} + + steps: + - name: ✅ Check comment + if: github.event_name == 'issue_comment' + run: | + if [[ "${{ github.event.comment.body }}" == "/validate-sgid-index" ]]; then + echo "Validating SGID Index" + else + echo "Skipping SGID Index validation" + exit 78 + fi + + - name: ⬇️ Set up code + uses: actions/checkout@v4 + with: + show-progress: false + + - name: ⎔ Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: lts/* + cache: npm + + - name: 📦 Install script dependencies + run: npm install + + - name: ✔ Running script + uses: gh640/command-result-action@v1 + id: validate + with: + command: node validate-sgid-index.mjs + cwd: ./src/scripts + + - name: 📝 Create issue + id: create-issue + if: steps.validate.outputs.exitCode != 0 + uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + filename: .github/SGID_INDEX_ISSUE_TEMPLATE.md + update_existing: true + + - name: Find Open Issues + uses: actions/github-script@v4 + id: find-issue + if: steps.validate.outputs.exitCode == 0 + with: + script: | + const { data: issues } = await github.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'sgid index validation' + }); + const issueNumber = issues[0]?.number; + console.log('issueNumber: ' + issueNumber); + + return issueNumber; + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Find Comment + uses: peter-evans/find-comment@v3 + if: steps.create-issue.outputs.number || steps.find-issue.outputs.result + id: find-comment + with: + issue-number: ${{ steps.create-issue.outputs.number || steps.find-issue.outputs.result }} + comment-author: github-actions[bot] + body-includes: Validation Output + + - name: ✍️ Updating issue comment + if: steps.create-issue.outputs.number || steps.find-issue.outputs.result + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ steps.create-issue.outputs.number || steps.find-issue.outputs.result }} + comment-id: ${{ steps.find-comment.outputs.comment-id }} + edit-mode: replace + body: | + ### SGID Index Validation Output + + #### Results + ``` + ${{ steps.validate.outputs.stdout }} + ``` + + #### Errors + ${{ steps.validate.outputs.stderr }} + + - name: 🚦 Check for errors + if: steps.validate.outputs.exitCode != 0 + run: | + echo "::error::Validate stderr${{ steps.validate.outputs.stderr }}" + exit ${{ steps.validate.outputs.exitCode }} + + - name: 🎉 Close issue + if: steps.validate.outputs.exitCode == 0 && steps.find-issue.outputs.result + run: gh issue close --comment "All validations have passed successfully" "${{ steps.find-issue.outputs.result }}" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.vscode/settings.json b/.vscode/settings.json index 96aad38bbf..f20525b3db 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -260,6 +260,7 @@ "oil", "oiland", "opendata", + "opensgid", "Oquirrh", "Orangeville", "Orderville", diff --git a/src/data/downloadMetadata.ts b/src/data/downloadMetadata.ts index f0c34eab8d..3165ec7b34 100644 --- a/src/data/downloadMetadata.ts +++ b/src/data/downloadMetadata.ts @@ -78,7 +78,7 @@ export const dataPages: DownloadMetadata = { itemId: 'abd5d645b0e144558d0ab15e1f043490', name: 'BLM Wilderness Study Areas for Utah', featureServiceId: 'Wilderness_BLMWSAs', - openSgid: 'boundaries.blm_wilderness_study_areas_for_utah', + openSgid: 'planning.blm_wilderness_study_areas_for_utah', layerId: 0, }, 'Utah Navajo Chapters': { @@ -740,7 +740,7 @@ export const dataPages: DownloadMetadata = { openSgid: 'energy.oil_gas_fields', layerId: 0, featureServiceHost: 'https://services.arcgis.com/ZzrwjTRez6FJiOq4/ArcGIS/rest/services/', - oddHubName: 'utah-oil-and-gas-fields' + oddHubName: 'utah-oil-and-gas-fields', }, 'Utah Energy Corridor Areas': { itemId: '3c39e10f78c34cd981f28bde8ad76e14', @@ -1267,9 +1267,11 @@ export const dataPages: DownloadMetadata = { layerId: 0, }, 'Utah Quaternary Faults': { - itemId: 'hosted by UGS', + itemId: '9c85978d0fb54570bc60bec467e2fa7f', name: 'Utah Quaternary Faults', - featureServiceId: 'QuaternaryFaults', + featureServiceHost: 'https://services.arcgis.com/ZzrwjTRez6FJiOq4/arcgis', + featureServiceId: 'Utah_Quaternary_Faults_20201207', + externalHubOrganization: 'utahDNR', openSgid: 'geoscience.quaternary_faults', layerId: 0, }, @@ -1878,7 +1880,7 @@ export const dataPages: DownloadMetadata = { 'Utah US Congress Districts 2022 to 2032': { itemId: 'b5ef0b24f994467fb4f22b27b1a47e25', name: 'Utah US Congress Districts 2022 to 2032', - featureServiceId: 'USCongressDistricts2022to2032 ', + featureServiceId: 'USCongressDistricts2022to2032', openSgid: 'political.us_congress_districts_2022_to_2032', layerId: 0, }, @@ -2088,14 +2090,14 @@ export const dataPages: DownloadMetadata = { 'Utah Broadband Service': { itemId: '2b479a30791c445eb135e05acf77dbcc', name: 'Utah Broadband Service', - featureServiceId: 'BroadBandService', + featureServiceId: 'BroadbandService', openSgid: 'utilities.broadband_service', layerId: 0, }, 'Utah Streams NHD': { itemId: 'd9b5ac9220ff415994b193c9ce022f86', name: 'Utah Streams NHD', - featureServiceId: 'StreamsNHDHighRes', + featureServiceId: 'UtahStreamsNHD', openSgid: 'water.streams_nhd', layerId: 0, }, @@ -2144,7 +2146,7 @@ export const dataPages: DownloadMetadata = { 'Utah Watersheds Area': { itemId: 'eedaf451685b4cbe875ecf692e1b54ff', name: 'Utah Watersheds Area', - featureServiceId: 'Watersheds_Area', + featureServiceId: 'UtahWatershedsArea', openSgid: 'water.watersheds_area', layerId: 0, }, @@ -2368,8 +2370,8 @@ export const dataPages: DownloadMetadata = { 'Utah Health Small Statistical Areas Obesity and Activity': { itemId: '5c5c709041d343f18096b7a356831be9', name: 'Utah Health Small Statistical Areas Obesity and Activity', - featureServiceId: null, - openSgid: '', + featureServiceId: 'HealthSmallStatisticalAreas2020', + openSgid: 'health.health_small_statistical_areas_2020', layerId: 0, }, 'Utah House Districts 2022 to 2032': { @@ -2424,7 +2426,7 @@ export const dataPages: DownloadMetadata = { 'Utah School Board Districts 2022 to 2032': { itemId: 'ebb7b8e8a0104b448d3bbf3ee82a5580', name: 'Utah School Board Districts 2022 to 2032', - featureServiceId: 'UtahSchoolBoardDistricts2022to2032 ', + featureServiceId: 'UtahSchoolBoardDistricts2022to2032', openSgid: 'political.school_board_districts_2022_to_2032', layerId: 0, }, @@ -2571,7 +2573,7 @@ export const dataPages: DownloadMetadata = { 'Utah Senate Districts 2022 to 2032': { itemId: '5289f23d9e76421f89d9a9ef4864937a', name: 'Utah Senate Districts 2022 to 2032', - featureServiceId: 'UtahSenateDistricts2022to2032 ', + featureServiceId: 'UtahSenateDistricts2022to2032', openSgid: 'political.senate_districts_2022_to_2032', layerId: 0, }, @@ -2579,7 +2581,7 @@ export const dataPages: DownloadMetadata = { itemId: '67edfe9ded464e3393c1932b4d39a6b2', name: 'Utah Health Small Statistical Areas 2020', featureServiceId: 'HealthSmallStatisticalAreas2020', - openSgid: 'health_small_statistical_areas_2020', + openSgid: 'health.health_small_statistical_areas_2020', layerId: 0, }, 'Utah Tax Areas 2021': { diff --git a/src/scripts/.gitignore b/src/scripts/.gitignore new file mode 100644 index 0000000000..16d3c4dbbf --- /dev/null +++ b/src/scripts/.gitignore @@ -0,0 +1 @@ +.cache diff --git a/src/scripts/package-lock.json b/src/scripts/package-lock.json index 1d0a89bf89..a010ceb3fe 100644 --- a/src/scripts/package-lock.json +++ b/src/scripts/package-lock.json @@ -9,7 +9,15 @@ "dependencies": { "fast-xml-parser": "^4.3.5", "glob": "^10.3.10", - "ky": "^1.2.2" + "google-auth-library": "^9.7.0", + "google-spreadsheet": "^4.1.1", + "json-to-markdown-table": "^1.0.0", + "knex": "^3.1.0", + "ky": "^1.2.2", + "pg": "^8.11.3", + "progress": "^2.0.3", + "ts-import": "^5.0.0-beta.0", + "uuid": "^9.0.1" } }, "node_modules/@isaacs/cliui": { @@ -83,6 +91,17 @@ "node": ">=14" } }, + "node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -102,11 +121,53 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", + "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "engines": { + "node": "*" + } + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -115,6 +176,19 @@ "balanced-match": "^1.0.0" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -131,6 +205,38 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/colorette": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", + "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "engines": { + "node": ">=14" + } + }, + "node_modules/comment-parser": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/comment-parser/-/comment-parser-1.3.1.tgz", + "integrity": "sha512-B52sN2VNghyq5ofvUsqZjmk6YkihBX5vMSChmSK9v4ShjKf3Vk5Xcmgpw4o+iIgtrnM/u5FiMpz9VKb8lpBveA==", + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -144,16 +250,69 @@ "node": ">= 8" } }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "node_modules/escalade": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/fast-xml-parser": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.5.tgz", @@ -175,6 +334,25 @@ "fxparser": "src/cli/cli.js" } }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/foreground-child": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", @@ -190,6 +368,66 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.3.0.tgz", + "integrity": "sha512-p+ggrQw3fBwH2F5N/PAI4k/G/y1art5OxKpb2J2chwNNHM4hHuAOtivjPuirMF4KNKwTTUal/lPfL2+7h2mEcg==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/getopts": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", + "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==" + }, "node_modules/glob": { "version": "10.3.10", "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", @@ -211,6 +449,93 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/google-auth-library": { + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.7.0.tgz", + "integrity": "sha512-I/AvzBiUXDzLOy4iIZ2W+Zq33W4lcukQv1nl7C8WUA6SQwyQwUwu3waNmWNAvzds//FG8SZ+DnKnW/2k6mQS8A==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-spreadsheet": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/google-spreadsheet/-/google-spreadsheet-4.1.1.tgz", + "integrity": "sha512-Npk/xAMTgxEt/m/X9EXIqdY6CEYGiqUHrSuiLnNSKli5H+wiOQLSLsnfMxcdNPH6aSh6GttZm6QJhrnsxjwpZQ==", + "dependencies": { + "axios": "^1.4.0", + "lodash": "^4.17.21" + }, + "peerDependencies": { + "google-auth-library": "^8.8.0 || ^9.0.0" + }, + "peerDependenciesMeta": { + "google-auth-library": { + "optional": true + } + } + }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-core-module": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "dependencies": { + "hasown": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -219,6 +544,17 @@ "node": ">=8" } }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -241,6 +577,91 @@ "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-to-markdown-table": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-to-markdown-table/-/json-to-markdown-table-1.0.0.tgz", + "integrity": "sha512-doujwoq5AsxYhumxg+KfkuNWy7Ch7nEWmCC+5UykGm4ommJBD52oqexL7625ZK0bddlDV4fhEkX+m0j8h2n8Pw==", + "dependencies": { + "lodash": "^4.16.4" + } + }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/knex": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/knex/-/knex-3.1.0.tgz", + "integrity": "sha512-GLoII6hR0c4ti243gMs5/1Rb3B+AjwMOfjYm97pu0FOQa7JH56hgBxYf5WK2525ceSbBY1cjeZ9yk99GPMB6Kw==", + "dependencies": { + "colorette": "2.0.19", + "commander": "^10.0.0", + "debug": "4.3.4", + "escalade": "^3.1.1", + "esm": "^3.2.25", + "get-package-type": "^0.1.0", + "getopts": "2.3.0", + "interpret": "^2.2.0", + "lodash": "^4.17.21", + "pg-connection-string": "2.6.2", + "rechoir": "^0.8.0", + "resolve-from": "^5.0.0", + "tarn": "^3.0.2", + "tildify": "2.0.0" + }, + "bin": { + "knex": "bin/cli.js" + }, + "engines": { + "node": ">=16" + }, + "peerDependenciesMeta": { + "better-sqlite3": { + "optional": true + }, + "mysql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "pg-native": { + "optional": true + }, + "sqlite3": { + "optional": true + }, + "tedious": { + "optional": true + } + } + }, "node_modules/ky": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/ky/-/ky-1.2.2.tgz", @@ -252,6 +673,11 @@ "url": "https://github.com/sindresorhus/ky?sponsor=1" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "node_modules/lru-cache": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", @@ -260,6 +686,25 @@ "node": "14 || >=16.14" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", @@ -282,6 +727,40 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/options-defaults": { + "version": "2.0.40", + "resolved": "https://registry.npmjs.org/options-defaults/-/options-defaults-2.0.40.tgz", + "integrity": "sha512-a0oW0AMaP/Uqk1gU7s3unE83wzs/MACy3wsCnNREn4wqp4KCcxRdulRjf0d2FeIxENbGJ4EBGtHTQ6J30XB6Cw==" + }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -290,6 +769,11 @@ "node": ">=8" } }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, "node_modules/path-scurry": { "version": "1.10.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz", @@ -305,6 +789,191 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.1.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -335,6 +1004,14 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -390,6 +1067,98 @@ "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tarn": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/tildify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", + "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/ts-import": { + "version": "5.0.0-beta.0", + "resolved": "https://registry.npmjs.org/ts-import/-/ts-import-5.0.0-beta.0.tgz", + "integrity": "sha512-YOe/xCmwDWughfeaAaGJ4UWzlCKNnt9e+oda3St6mUMkRJCTBhBso+7XApIijw7Mr9SS6NLOdav8i5EJrx7UVQ==", + "dependencies": { + "comment-parser": "1.3.1", + "options-defaults": "2.0.40", + "tslib": "2.5.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "typescript": "5" + } + }, + "node_modules/tslib": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", + "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + }, + "node_modules/typescript": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.3.tgz", + "integrity": "sha512-KrPd3PKaCLr78MalgiwJnA25Nm8HAmdwN3mYUYZgG/wizIo9EainNVQI9/yDavtVFRN2h3k8uf3GLHuhDMgEHg==", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -496,6 +1265,14 @@ "funding": { "url": "https://github.com/chalk/strip-ansi?sponsor=1" } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } } } } diff --git a/src/scripts/package.json b/src/scripts/package.json index 7c7848ba64..806a04785a 100644 --- a/src/scripts/package.json +++ b/src/scripts/package.json @@ -3,6 +3,14 @@ "dependencies": { "fast-xml-parser": "^4.3.5", "glob": "^10.3.10", - "ky": "^1.2.2" + "google-auth-library": "^9.7.0", + "google-spreadsheet": "^4.1.1", + "json-to-markdown-table": "^1.0.0", + "knex": "^3.1.0", + "ky": "^1.2.2", + "pg": "^8.11.3", + "progress": "^2.0.3", + "ts-import": "^5.0.0-beta.0", + "uuid": "^9.0.1" } } diff --git a/src/scripts/utilities.mjs b/src/scripts/utilities.mjs index c9e283c21d..6375d85604 100644 --- a/src/scripts/utilities.mjs +++ b/src/scripts/utilities.mjs @@ -1,3 +1,4 @@ +import knex from 'knex'; import ky from 'ky'; export async function validateUrl(url) { @@ -48,3 +49,95 @@ export async function validateUrl(url) { valid: true, }; } + +let dbClient; +export async function validateOpenSgidTableName(table, schema) { + if (!dbClient) { + dbClient = knex({ + client: 'postgres', + connection: { + host: 'opensgid.agrc.utah.gov', + user: 'agrc', + password: 'agrc', + port: 5432, + database: 'opensgid', + }, + }); + } + + const errors = []; + + const tableResult = await dbClient.raw(`SELECT 1 FROM information_schema.tables WHERE table_name = '${table}'`); + const tableFound = tableResult.rows.length === 1; + if (!tableFound) { + errors.push(`"${schema}.${table}" Open SGID table not found`); + } + + const schemaResult = await dbClient.raw(`SELECT 1 FROM information_schema.schemata WHERE schema_name = '${schema}'`); + const schemaFound = schemaResult.rows.length === 1; + if (!schemaFound) { + errors.push(`"${schema}.${table}" Open SGID schema not found`); + } + + if (errors.length > 0) { + return { + valid: false, + message: errors.join(','), + }; + } + + return { + valid: true, + }; +} + +export async function validateOpenDataUrl(url) { + const domainToOrg = { + 'data.wfrc.org': 'wfrc', + 'opendata.gis.utah.gov': 'utah', + 'data-uplan.opendata.arcgis.com': 'uplan', + 'dwr-data-utahdnr.hub.arcgis.com': 'utahDNR', + }; + + const match = /https:\/\/.*?\/datasets\/((?.*)::)?(?.*?)(\/|$)/.exec(url); + const slug = match.groups.slug; + let org = match.groups.org; + + const domain = new URL(url).hostname; + if (!Object.keys(domainToOrg).includes(domain)) { + throw new Error(`Unknown domain: ${domain}`); + } + + if (!org) { + org = domainToOrg[domain]; + } + const openDataQuery = 'https://opendata.arcgis.com/api/v3/datasets'; + + let response; + try { + response = await ky(openDataQuery, { + searchParams: { + 'fields[datasets]': 'slug,boundary,extent,recordCount,searchDescription,statistics', + 'filter[slug]': `${org}::${slug}`, + }, + }).json(); + } catch (error) { + return { + valid: false, + message: `request error: ${error.message}`, + }; + } + + if (response.data.length === 0) { + return { + valid: false, + message: 'open data item not found', + }; + } + + return { + valid: true, + data: response.data[0], + message: 'valid open data url', + }; +} diff --git a/src/scripts/utilities.test.mjs b/src/scripts/utilities.test.mjs index 362e649f02..0aa0feb03c 100644 --- a/src/scripts/utilities.test.mjs +++ b/src/scripts/utilities.test.mjs @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; -import { validateUrl } from './utilities.mjs'; +import { validateOpenDataUrl, validateOpenSgidTableName, validateUrl } from './utilities.mjs'; describe('validateUrl', () => { it('handles a valid url', async () => { @@ -32,3 +32,38 @@ describe('validateUrl', () => { assert.match(result.message, /failed request/); }); }); + +describe('validateOpenSgidTableName', () => { + it('handles a valid table', async () => { + const result = await validateOpenSgidTableName('county_boundaries', 'boundaries'); + assert(result.valid); + }); + + it('handles an invalid table', async () => { + const result = await validateOpenSgidTableName('bad_table', 'boundaries'); + assert(!result.valid); + assert.match(result.message, /table/); + }); + + it('handles an invalid schema', async () => { + const result = await validateOpenSgidTableName('county_boundaries', 'schema'); + assert(!result.valid); + assert.match(result.message, /schema/); + }); +}); + +describe('validateOpenDataUrl', () => { + const tests = [ + ['https://opendata.gis.utah.gov/datasets/utah::utah-blm-monuments-and-ncas', true], + ['https://data.wfrc.org/datasets/access-to-opportunities-work-related-taz-based/about', false], + ['https://opendata.gis.utah.gov/datasets/utahDNR::utah-geochronology', true], + ['https://data-uplan.opendata.arcgis.com/datasets/functional-class-alrs', false], + ]; + + for (const test of tests) { + it(`should return ${test[1]} for ${test[0]}`, async () => { + const result = await validateOpenDataUrl(test[0]); + assert.equal(result.valid, test[1]); + }); + } +}); diff --git a/src/scripts/validate-sgid-index.mjs b/src/scripts/validate-sgid-index.mjs new file mode 100644 index 0000000000..e504f98b2b --- /dev/null +++ b/src/scripts/validate-sgid-index.mjs @@ -0,0 +1,297 @@ +import { GoogleAuth, auth } from 'google-auth-library'; +import { GoogleSpreadsheet } from 'google-spreadsheet'; +import jsonToMarkdown from 'json-to-markdown-table'; +import ky from 'ky'; +import ProgressBar from 'progress'; +import * as tsImport from 'ts-import'; +import { v4 as uuid } from 'uuid'; +import { validateOpenDataUrl, validateOpenSgidTableName, validateUrl } from './utilities.mjs'; + +const downloadMetadata = await tsImport.load('../data/downloadMetadata.ts'); + +const spreadsheetId = '11ASS7LnxgpnD0jN4utzklREgMf1pcvYjcXcIcESHweQ'; +const ourWebSite = 'https://gis.utah.gov'; + +const scopes = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive']; +let client; +if (process.env.GITHUB_ACTIONS) { + console.log('using ci credentials'); + client = auth.fromJSON(JSON.parse(process.env.GOOGLE_PRIVATE_KEY)); + client.scopes = scopes; +} else { + client = new GoogleAuth({ + scopes, + }); +} + +console.log('loading spreadsheet'); +const spreadsheet = new GoogleSpreadsheet(spreadsheetId, client); +await spreadsheet.loadInfo(); +const worksheet = spreadsheet.sheetsByTitle['SGID Index']; + +const fieldNames = { + displayName: 'displayName', + hubName: 'hubName', + id: 'id', + itemId: 'itemId', + openSgid: 'openSgid', + openSgidTableName: 'openSgidTableName', + productPage: 'productPage', + serverLayerId: 'serverLayerId', + ugrcStatus: 'ugrcStatus', +}; + +function getFieldName(name) { + const fieldName = fieldNames[name]; + + if (!fieldName) { + throw new Error(`field name "${name}" not found`); + } + + return fieldName; +} + +const errors = []; +function recordError(message, row) { + errors.push({ + displayName: row.get(getFieldName('displayName')), + id: row.get(getFieldName('id')), + Error: message, + }); +} + +function toBoolean(value) { + return value === 'TRUE'; +} + +async function openSGIDTableName(row) { + if (!toBoolean(row.get(getFieldName('openSgid')))) { + return; + } + + const cellValue = row.get(getFieldName('openSgidTableName')); + if (!cellValue) { + recordError('openSgidTableName is empty', row); + return; + } + + const [schema, tableName] = cellValue.split('.'); + const result = await validateOpenSgidTableName(tableName, schema); + + if (!result.valid) { + recordError(result.message, row); + } +} + +function trimFields(row) { + let changed = false; + + for (const field in fieldNames) { + const cellValue = row.get(getFieldName(field)); + if (cellValue && cellValue.trim() !== cellValue) { + row.set(getFieldName(field), cellValue.trim()); + changed = true; + } + } + + return changed; +} + +async function productPage(row) { + const cellValue = row.get(getFieldName('productPage')); + + if (!cellValue) { + // productPage is not a required field + return; + } + + const url = cellValue.startsWith('/') ? `${ourWebSite}${cellValue}` : cellValue; + + let result = await validateUrl(url); + + if (result.valid && /\/datasets\//.test(url)) { + result = await validateOpenDataUrl(url); + } + + if (!result.valid) { + recordError(`productPage: ${result.message}`, row); + } +} + +async function idGuid(row) { + const cellValue = row.get(getFieldName('id')); + + if (!cellValue) { + row.set('id', uuid()); + + return true; + } +} + +async function itemId(row) { + const cellValue = row.get(getFieldName('itemId')); + let layerId = row.get(getFieldName('serverLayerId')); + if (!layerId) { + layerId = 0; + } + + if (!cellValue) { + // TODO: internal datasets _should_ have an itemId + // also, check for deprecated and openSgid/arcGisOnline field values + // itemId is not a required field + return; + } + + if (cellValue.length !== 32 || cellValue.indexOf(' ') !== -1) { + recordError('itemId is not a valid AGOL item id', row); + return; + } + + let hubData; + try { + hubData = await ky(`https://opendata.arcgis.com/api/v3/datasets/${cellValue}_${layerId}`).json(); + } catch (error) { + recordError(`itemId hub request error: ${error.message}`, row); + return; + } + const serviceParts = hubData.data.attributes.url.split('/rest/services/'); + + const newData = { + hubName: hubData.data.attributes.name, + hubOrganization: hubData.data.attributes.slug.split('::')[0], + serverHost: serviceParts[0], + serverServiceName: serviceParts[1].split(/\/(FeatureServer|MapServer)\//)[0], + serverLayerId: layerId, + }; + + let changed = false; + for (const field in newData) { + if (row.get(field) !== newData[field]) { + row.set(field, newData[field]); + changed = true; + } + } + + return changed; +} + +async function duplicates(row) { + for (const field in duplicateLookups) { + const value = row.get(field); + if (!value) { + continue; + } + + if (duplicateLookups[field][value]?.length > 1) { + recordError(`duplicate ${field}: "${value}" found`, row); + } + } +} + +async function downloadMetadataCheck(row) { + const name = row.get(getFieldName('hubName')); + const metadata = downloadMetadata.dataPages[name]; + + if (!metadata || metadata.featureServiceId === null) { + return; + } + + const metadataChecks = [ + // sgid index field, metadata field + ['itemId', 'itemId'], + ['hubName', 'name'], + ['serverServiceName', 'featureServiceId'], + ['openSgidTableName', 'openSgid'], + ['serverLayerId', 'layerId'], + ]; + + for (const [sgidIndexField, metadataField] of metadataChecks) { + const sgidIndexValue = row.get(sgidIndexField).toString(); + const metadataValue = metadata[metadataField].toString(); + + if (sgidIndexValue !== metadataValue) { + recordError( + `downloadMetadata(${name}): "${metadataField}" does not match SGID Index column "${sgidIndexField}"`, + row, + ); + } + } +} + +const duplicateLookups = { + openSgidTableName: {}, + itemId: {}, + id: {}, + displayName: {}, +}; +function buildDuplicateLookups(rows) { + console.log('building duplicate lookups'); + + for (const field in duplicateLookups) { + for (const row of rows) { + const value = row.get(field); + if (!value) { + continue; + } + + if (duplicateLookups[field][value]) { + duplicateLookups[field][value].push(row); + } else { + duplicateLookups[field][value] = [row]; + } + } + } +} + +const checks = [ + // these functions must return a promise + openSGIDTableName, + productPage, + idGuid, + itemId, + duplicates, + downloadMetadataCheck, +]; + +const rows = await worksheet.getRows(); +buildDuplicateLookups(rows); + +let updatedRowsCount = 0; +console.log(`checking ${rows.length} rows`); +const progressBar = new ProgressBar(':bar :percent ETA: :etas ', { total: rows.length }); +const skipStatuses = ['deprecated', 'shelved']; +for (const row of rows) { + progressBar.tick(); + + if (skipStatuses.includes(row.get(getFieldName('ugrcStatus'))?.toLowerCase())) { + continue; + } + + let changed; + try { + changed = trimFields(row); // we want trim to run first + + const checksChanged = (await Promise.all(checks.map((check) => check(row)))).some((result) => result); + + changed = changed || checksChanged; + } catch (error) { + recordError(error.message, row); + } + + if (changed) { + await row.save(); + updatedRowsCount++; + } +} + +if (errors.length > 0) { + errors.sort((a, b) => a.displayName.localeCompare(b.displayName)); + console.error(jsonToMarkdown(errors, ['displayName', 'id', 'Error'])); + console.log(`\ntotal errors: ${errors.length}`); + console.log(`updated ${updatedRowsCount} rows`); + process.exit(1); +} + +console.log('All rows validated successfully'); +console.log(`updated ${updatedRowsCount} rows`); +process.exit(0);