From 742a3b138d6b105dd78ecdce31f1f56c06749d81 Mon Sep 17 00:00:00 2001 From: David Taylor Date: Mon, 13 May 2024 13:13:04 +0100 Subject: [PATCH] Add GoogleOther crawler user agent (#27000) Per https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers > GoogleOther is the generic crawler that may be used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development. This commit will ensure it's served the crawler view, and included in crawler metrics --- config/site_settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/site_settings.yml b/config/site_settings.yml index 9db11a7bc2c..710b194cc4a 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -1958,7 +1958,7 @@ security: list_type: compact crawler_user_agents: hidden: true - default: "rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor|lighthouse|google-inspectiontool|gptbot|claudebot|anthropic-ai|brightbot" + default: "rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor|lighthouse|google-inspectiontool|gptbot|claudebot|anthropic-ai|brightbot|googleother" type: list list_type: compact browser_update_user_agents: