Introduce new script to check external links
This CL introduces "npm run check-external-links". The check parses
all string literals in first party JS/TS code for Urls. All the Urls
are then "pinged" by making a HEAD request to verify they still point
to an active resource. Example output:
$ time npm run check-external-links
> chrome-devtools-frontend@ check-external-links /usr/local/google/home/szuend/dev/devtools/devtools-frontend
> third_party/node/node.py --output scripts/check_external_links.js
Collecting JS/TS source files ... 1019 files found.
Collecting Urls from files ...248 unique Urls found.
Sending a HEAD request to each one ...
All Urls are accessible and point to existing resources.
npm run check-external-links 9.21s user 0.40s system 149% cpu 6.415 total
Please note that we can't make this check part of our PRESUBMIT, as
it makes external requests.
Also note that this check can't be implemented as an ESLint rule,
as ESLint rules are not allowed to have an async workload.
R=aerotwist@chromium.org, sigurds@chromium.org
Bug: chromium:1170310
Change-Id: I064161dd9038143d2e360c1716f8fb622dc119ee
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/2659021
Commit-Queue: Simon Zünd <szuend@chromium.org>
Reviewed-by: Sigurd Schneider <sigurds@chromium.org>
Reviewed-by: Paul Lewis <aerotwist@chromium.org>
diff --git a/scripts/check_external_links.js b/scripts/check_external_links.js
new file mode 100644
index 0000000..a2d9bbf
--- /dev/null
+++ b/scripts/check_external_links.js
@@ -0,0 +1,150 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+'use strict';
+
+const fs = require('fs');
+const https = require('https');
+const path = require('path');
+const ts = require('typescript');
+
+const readDirAsync = fs.promises.readdir;
+const readFileAsync = fs.promises.readFile;
+
+const ORIGIN_PATTERNS_TO_CHECK = [
+ new RegExp('^https://web.dev'),
+ new RegExp('^https://developers.google.com'),
+ new RegExp('^https://developer[s]?.chrome.com'),
+];
+
+const DIRECTORIES_TO_CHECK = [
+ 'front_end',
+];
+
+const EXCLUDE_DIRECTORIES = [
+ 'front_end/third_party',
+];
+
+const REQUEST_TIMEOUT = 5000;
+
+const REDIRECTS_CONSIDERED_ERROR = new Set([
+ /* Multiple Choices */ 300,
+ /* Moved permanently */ 301,
+ /* Permament redirect */ 308,
+]);
+
+const ROOT_REPOSITORY_PATH = path.resolve(__dirname, '..');
+const DIRECTORIES_TO_CHECK_PATHS = DIRECTORIES_TO_CHECK.map(directory => path.resolve(ROOT_REPOSITORY_PATH, directory));
+
+async function findAllSourceFiles(directory) {
+ if (EXCLUDE_DIRECTORIES.includes(path.relative(ROOT_REPOSITORY_PATH, directory))) {
+ return [];
+ }
+
+ const dirEntries = await readDirAsync(directory, {withFileTypes: true});
+ const files = await Promise.all(dirEntries.map(dirEntry => {
+ const resolvedPath = path.resolve(directory, dirEntry.name);
+ if (dirEntry.isDirectory()) {
+ return findAllSourceFiles(resolvedPath);
+ }
+ if (dirEntry.isFile() && /\.(js|ts)$/.test(dirEntry.name)) {
+ return resolvedPath;
+ }
+ return []; // Let Array#flat filter out files we are not interested in.
+ }));
+ return files.flat();
+}
+
+function collectUrlsToCheck(node) {
+ const nodesToVisit = [node];
+ const urlsToCheck = [];
+ while (nodesToVisit.length) {
+ const currentNode = nodesToVisit.shift();
+ if (currentNode.kind === ts.SyntaxKind.StringLiteral ||
+ currentNode.kind === ts.SyntaxKind.NoSubstitutionTemplateLiteral) {
+ const checkUrl = ORIGIN_PATTERNS_TO_CHECK.some(originPattern => originPattern.test(currentNode.text));
+ if (checkUrl) {
+ urlsToCheck.push(currentNode.text);
+ }
+ }
+ nodesToVisit.push(...currentNode.getChildren());
+ }
+ return urlsToCheck;
+}
+
+async function collectUrlsToCheckFromFile(filePath) {
+ const content = await readFileAsync(filePath, 'utf8');
+ const sourceFile = ts.createSourceFile(filePath, content, ts.ScriptTarget.ESNext, true);
+ return collectUrlsToCheck(sourceFile);
+}
+
+async function checkUrls(urls) {
+ // clang-format off
+ const requestPromises = urls.map(url => new Promise(resolve => {
+ const request = https.request(url, {method: 'HEAD'}, response => {
+ resolve({url, statusCode: response.statusCode});
+ });
+
+ request.on('error', err => {
+ resolve({url, error: err});
+ });
+ request.setTimeout(REQUEST_TIMEOUT, _ => {
+ resolve({url, error: `Timed out after ${REQUEST_TIMEOUT}`});
+ });
+ request.end();
+ }));
+ // clang-format on
+
+ return Promise.all(requestPromises);
+}
+
+function includeRequestResultInOutput(requestResult) {
+ return requestResult.error || requestResult.statusCode !== 200;
+}
+
+function isErrorStatusCode(statusCode) {
+ return statusCode >= 400 || REDIRECTS_CONSIDERED_ERROR.has(statusCode);
+}
+
+function requestResultIsErronous(requestResult) {
+ return requestResult.error || isErrorStatusCode(requestResult.statusCode);
+}
+
+function printSelectedRequestResults(requestResults) {
+ const requestsToPrint = requestResults.filter(includeRequestResultInOutput);
+ if (requestsToPrint.length === 0) {
+ console.log('\nAll Urls are accessible and point to existing resources.\n');
+ return;
+ }
+
+ for (const requestResult of requestsToPrint) {
+ if (requestResult.error) {
+ console.error(`[Failure] ${requestResult.error} - ${requestResult.url}`);
+ } else if (isErrorStatusCode(requestResult.statusCode)) {
+ console.error(`[Failure] Status Code: ${requestResult.statusCode} - ${requestResult.url}`);
+ } else {
+ console.log(`Status Code: ${requestResult.statusCode} - ${requestResult.url}`);
+ }
+ }
+}
+
+async function main() {
+ process.stdout.write('Collecting JS/TS source files ... ');
+ const sourceFiles = (await Promise.all(DIRECTORIES_TO_CHECK_PATHS.map(findAllSourceFiles))).flat();
+ process.stdout.write(`${sourceFiles.length} files found.\n`);
+
+ process.stdout.write('Collecting Urls from files ... ');
+ const urlsToCheck = (await Promise.all(sourceFiles.map(collectUrlsToCheckFromFile))).flat();
+ const deduplicatedUrlsToCheck = new Set(urlsToCheck);
+ process.stdout.write(`${deduplicatedUrlsToCheck.size} unique Urls found.\n`);
+
+ process.stdout.write('Sending a HEAD request to each one ...\n');
+ const requestResults = await checkUrls([...deduplicatedUrlsToCheck]);
+ printSelectedRequestResults(requestResults);
+
+ const exitCode = requestResults.some(requestResultIsErronous) ? 1 : 0;
+ process.exit(exitCode);
+}
+
+main();