From eac2211ea751e9f27198e57850e3de47857325d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Go=C5=82e=CC=A8biowski-Owczarek?= Date: Sat, 12 Apr 2025 13:16:42 +0200 Subject: [PATCH] Build: Skip checking oembed wp-json links in the hydra crawler The oembed endpoint responds HTTP 429 Too Many Requests too often; perhaps because almost every page links to one. Also, run the spider check on PRs that modify the hydra config file. --- .github/configs/hydra-config.json | 7 ++++++- .github/workflows/spider-check.yaml | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/configs/hydra-config.json b/.github/configs/hydra-config.json index 3d367e7b..2173722b 100644 --- a/.github/configs/hydra-config.json +++ b/.github/configs/hydra-config.json @@ -1,5 +1,10 @@ { + "//": [ + "2023-05: twitter.com serves broken redirect-loop", + "2025-04: The oembed endpoint responds HTTP 429 Too Many Requests too often; perhaps because almost every page links to one" + ], "exclude_scheme_prefixes": [ - "https://twitter.com/" + "https://twitter.com/", + "https://api.jquery.com/wp-json/oembed/1.0/embed" ] } diff --git a/.github/workflows/spider-check.yaml b/.github/workflows/spider-check.yaml index 378da3db..95ea1ea2 100644 --- a/.github/workflows/spider-check.yaml +++ b/.github/workflows/spider-check.yaml @@ -9,9 +9,11 @@ on: push: paths: - .github/workflows/spider-check.yaml + - .github/configs/hydra-config.json pull_request: paths: - .github/workflows/spider-check.yaml + - .github/configs/hydra-config.json jobs: spider-check: