diff --git a/bin/crawl-api-links.js b/bin/crawl-api-links.js index 7dae34ef6eedc9f..2e31f9c35f66ca9 100644 --- a/bin/crawl-api-links.js +++ b/bin/crawl-api-links.js @@ -21,6 +21,13 @@ async function checkLinks() { }); const page = await browser.newPage(); + // skip image requests + await page.setRequestInterception(true); + page.on("request", (request) => { + if (request.resourceType() === "image") request.abort(); + else request.continue(); + }); + const sitemapUrl = "https://developers.cloudflare.com/sitemap.xml"; await page.goto(sitemapUrl, { timeout: navigationTimeout }); @@ -51,23 +58,22 @@ async function checkLinks() { } if ( - pageLink.includes("developers.cloudflare.com/api/operations/") || - pageLink.startsWith("/api/operations/") + pageLink.includes("developers.cloudflare.com/api/resources/") || + pageLink.startsWith("/api/resources/") ) { console.log(`Evaluating link: ${pageLink}`); - await page.goto(pageLink, { + const response = await page.goto(pageLink, { waitUntil: "networkidle0", timeout: navigationTimeout, }); visitedLinks.push(pageLink); - const statusCode = await page.evaluate(() => { - return { - url: window.location.href, - }; - }); - if (statusCode.url === "https://developers.cloudflare.com/api/") { - brokenLinks.push(pageLink); + if (response) { + if (response.status() === 404) { + brokenLinks.push(pageLink); + } + } else { + console.log("WARNING: Didn't receive a response... skipping."); } } } diff --git a/package.json b/package.json index 38c85fecbe56505..a1c5fbed8932bc8 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "check:astro": "npm run sync && astro check", "check:functions": "npx tsc --noEmit -p ./functions/tsconfig.json", "check:worker": "npx tsc --noEmit -p ./worker/tsconfig.json", + "crawl-api-links": "node bin/crawl-api-links.js", "dev": "npx astro dev", "format": "npm run format:core && npm run format:data", "format:core": "npx prettier --write \"**/*.{js,jsx,ts,tsx,mjs,css}\"",