Files
jiang13-forum/services/crawler.go
freefire 3f50316ad0 fix: 完成 Gitea 目录改组收尾(import、构建与 LICENSE)
同步包路径与路由,去掉 SPA 构建步骤,对齐 Gitea 式 LICENSE,并更新规格/规则与占位 SSR。

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-29 04:01:20 +08:00

56 lines
1.0 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package services
import "strings"
// 常见搜索引擎 / 社交预览 / SEO 工具的 User-Agent 片段(小写匹配)
var seoCrawlerTokens = []string{
"googlebot",
"google-inspectiontool",
"bingbot",
"baiduspider",
"yandexbot",
"duckduckbot",
"slurp", // Yahoo
"sogou",
"bytespider",
"petalbot",
"applebot",
"facebookexternalhit",
"facebot",
"twitterbot",
"linkedinbot",
"discordbot",
"telegrambot",
"slackbot",
"whatsapp",
"preview", // 部分通用预览 UA
"embedly",
"quora link preview",
"pinterest",
"vkshare",
"w3c_validator",
"ahrefsbot",
"semrushbot",
"dotbot",
"mj12bot",
"gptbot",
"claudebot",
"anthropic-ai",
"chatgpt-user",
"oai-searchbot",
}
// IsSEOCrawler 是否为需要服务端 HTML 的爬虫 / 预览 bot动态渲染
func IsSEOCrawler(userAgent string) bool {
ua := strings.ToLower(strings.TrimSpace(userAgent))
if ua == "" {
return false
}
for _, token := range seoCrawlerTokens {
if strings.Contains(ua, token) {
return true
}
}
return false
}