fix: 拆除爬虫专用 HTML，并补齐 /favicon.ico 与 head 图标注入

人机统一走 SPA/首页 SSR；约定路径与 link rel=icon 便于爬虫识别站点图标。 Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-01 07:15:06 +08:00
parent cf7a37eccc
commit 4ca5c17e2e
6 changed files with 50 additions and 200 deletions
--- a/embed_static/embed.go
+++ b/embed_static/embed.go
@@ -14,9 +14,10 @@ import (
 var staticFS embed.FS

 var (
-	spaTitleRe      = regexp.MustCompile(`(?s)<title>.*?</title>`)
-	spaBrandTitleFn func() string
-	spaBrandJSONFn  func() []byte // 站点品牌 JSON，注入 window.__J13_BRANDING__
+	spaTitleRe        = regexp.MustCompile(`(?s)<title>.*?</title>`)
+	spaBrandTitleFn   func() string
+	spaBrandJSONFn    func() []byte // 站点品牌 JSON，注入 window.__J13_BRANDING__
+	spaBrandFaviconFn func() string // 站点 Favicon URL，注入 <link rel="icon">
 )

 // SetSPADocumentTitle 注册站点标题提供者，ServeSPA 会注入到入口 HTML，避免刷新闪烁
@@ -29,6 +30,11 @@ func SetSPABrandingJSON(fn func() []byte) {
 	spaBrandJSONFn = fn
 }

+// SetSPAFaviconURL 注册 Favicon URL 提供者，注入到入口 HTML 的 <link rel="icon">
+func SetSPAFaviconURL(fn func() string) {
+	spaBrandFaviconFn = fn
+}
+
 // SetupEmbed 配置内嵌资源：React SPA 静态资源
 func SetupEmbed(r *gin.Engine) error {
 	if sub, err := fs.Sub(staticFS, "static/spa/assets"); err == nil {
--- a/embed_static/spa_meta.go
+++ b/embed_static/spa_meta.go
@@ -62,13 +62,17 @@ func applySPAPageMeta(data []byte, meta *SPAPageMeta) []byte {
 		data = spaTitleRe.ReplaceAll(data, []byte("<title>"+escaped+"</title>"))
 	}

-	// —— 静态 SEO HTML（meta / OG / JSON-LD），紧跟 </title>，不经 JS ——
+	// —— 静态 SEO HTML（meta / OG / JSON-LD / favicon），紧跟 </title>，不经 JS ——
 	var seo strings.Builder
 	writeMeta(&seo, "description", meta.Description)
 	writeMeta(&seo, "keywords", meta.Keywords)
 	if canonical := strings.TrimSpace(meta.Canonical); canonical != "" {
 		seo.WriteString(`<link rel="canonical" href="` + html.EscapeString(canonical) + `"/>`)
 	}
+	if favicon := spaFaviconHref(); favicon != "" {
+		seo.WriteString(`<link rel="icon" href="` + html.EscapeString(favicon) + `"/>`)
+		seo.WriteString(`<link rel="shortcut icon" href="` + html.EscapeString(favicon) + `"/>`)
+	}
 	robots := strings.TrimSpace(meta.Robots)
 	if robots != "" {
 		writeMeta(&seo, "robots", robots)
@@ -187,3 +191,11 @@ func firstNonEmpty(vals ...string) string {
 	}
 	return ""
 }
+
+// spaFaviconHref 当前站点配置的 Favicon（相对或绝对 URL）
+func spaFaviconHref() string {
+	if spaBrandFaviconFn == nil {
+		return ""
+	}
+	return strings.TrimSpace(spaBrandFaviconFn())
+}
--- a/handler/seo.go
+++ b/handler/seo.go
@@ -2,18 +2,16 @@ package handler

 import (
 	"encoding/json"
-	"fmt"
-	"html"
 	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"

-	"github.com/gin-gonic/gin"
 	"git.iioio.com/freefire/jiang13-forum/embed_static"
 	"git.iioio.com/freefire/jiang13-forum/model"
 	"git.iioio.com/freefire/jiang13-forum/service"
+	"github.com/gin-gonic/gin"
 )

 var (
@@ -50,6 +48,18 @@ func (h *Handlers) RobotsTxt(c *gin.Context) {
 	c.Data(http.StatusOK, "text/plain; charset=utf-8", []byte(b.String()))
 }

+// FaviconICO 约定路径 /favicon.ico：有品牌图标则 302 到实际上传 URL，否则 404
+func (h *Handlers) FaviconICO(c *gin.Context) {
+	href := strings.TrimSpace(h.Settings.SiteBranding().Favicon)
+	if href == "" {
+		c.Status(http.StatusNotFound)
+		return
+	}
+	// 相对路径保持站内跳转；绝对 URL 也可 Redirect
+	c.Header("Cache-Control", "public, max-age=86400")
+	c.Redirect(http.StatusFound, href)
+}
+
 // SitemapXML 公开页面站点地图
 func (h *Handlers) SitemapXML(c *gin.Context) {
 	base := h.publicBaseURL(c)
@@ -147,9 +157,9 @@ func (h *Handlers) SitemapXML(c *gin.Context) {
 	c.Data(http.StatusOK, "application/xml; charset=utf-8", []byte(b.String()))
 }

-// ServePublicSPA 公开页入口：
-// - 普通用户：干净 SPA + <head> meta（无正文预渲染，避免刷新闪屏）
-// - 搜索/社交爬虫：服务端 HTML（动态渲染）
+// ServePublicSPA 公开页入口：人机统一响应，不再按 User-Agent 分叉。
+// - 首页 / 板块：完整首屏 SSR（serveFeedDocument）
+// - 其余公开页：干净 SPA + <head> meta
 // - 伪静态：按后台配置的后缀做规范 URL，非规范路径 301
 func (h *Handlers) ServePublicSPA(c *gin.Context) {
 	path := c.Request.URL.Path
@@ -181,11 +191,6 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
 		}
 	}

-	isBot := service.IsSEOCrawler(c.Request.UserAgent())
-	if isBot {
-		c.Header("Vary", "User-Agent")
-	}
-
 	// 板块首页（含可选伪静态后缀）
 	if bm := permalink.MatchBoardPath(path); bm.OK {
 		if bm.NeedsCanonicalRedirect(path) {
@@ -194,7 +199,7 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
 		}
 		board, err := h.Board.GetByID(bm.ID)
 		if err != nil {
-			h.serveNotFound(c, base, siteName, siteKeywords, path, isBot)
+			h.serveNotFound(c, base, siteName, siteKeywords, path)
 			return
 		}
 		desc := strings.TrimSpace(board.Description)
@@ -209,7 +214,6 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
 			OGType:      "website",
 			OGImage:     defaultImage,
 		}, siteName, siteKeywords)
-		// 板块首页：真人与爬虫共用完整首屏 HTML
 		h.serveFeedDocument(c, meta, board.ID)
 		return
 	}
@@ -222,14 +226,10 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
 		}
 		post, err := h.Post.FindByID(pm.ID)
 		if err != nil || !service.CanViewPost(post, h.currentUserID(c), h.isAdmin(c)) {
-			h.serveNotFound(c, base, siteName, siteKeywords, path, isBot)
+			h.serveNotFound(c, base, siteName, siteKeywords, path)
 			return
 		}
 		postKeywords := service.JoinSEOKeywords(post.Board.Name, siteKeywords)
-		if isBot {
-			c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(h.botPostHTML(base, siteName, defaultImage, postKeywords, post)))
-			return
-		}
 		embed_static.ServeSPAWithMeta(c, attachSiteSEO(h.postPageMeta(base, siteName, defaultImage, post), siteName, postKeywords))
 		return
 	}
@@ -242,11 +242,7 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
 		}
 		user, err := h.User.GetByID(um.ID)
 		if err != nil || user.Banned {
-			h.serveNotFound(c, base, siteName, siteKeywords, path, isBot)
-			return
-		}
-		if isBot {
-			c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(h.botUserHTML(base, siteName, defaultImage, siteKeywords, user)))
+			h.serveNotFound(c, base, siteName, siteKeywords, path)
 			return
 		}
 		embed_static.ServeSPAWithMeta(c, attachSiteSEO(h.userPageMeta(base, siteName, defaultImage, user), siteName, siteKeywords))
@@ -261,7 +257,7 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
 		}
 		page, err := h.SitePage.GetBySlug(pg.Slug, h.isAdmin(c))
 		if err != nil {
-			h.serveNotFound(c, base, siteName, siteKeywords, path, isBot)
+			h.serveNotFound(c, base, siteName, siteKeywords, path)
 			return
 		}
 		desc := service.ExcerptFromHTML(page.Content, seoDescMax)
@@ -273,18 +269,13 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
 			OGType:      "article",
 			OGImage:     defaultImage,
 		}, siteName, siteKeywords)
-		if isBot {
-			body := fmt.Sprintf(`<h1>%s</h1><div>%s</div>`, html.EscapeString(page.Title), page.Content)
-			c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(renderBotHTML(meta, body)))
-			return
-		}
 		embed_static.ServeSPAWithMeta(c, meta)
 		return
 	}

 	// 未知路径 → 404
 	if !isKnownPublicPath(path) {
-		h.serveNotFound(c, base, siteName, siteKeywords, path, isBot)
+		h.serveNotFound(c, base, siteName, siteKeywords, path)
 		return
 	}

@@ -297,12 +288,7 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
 	embed_static.ServeSPAWithMeta(c, meta)
 }

-func (h *Handlers) serveNotFound(c *gin.Context, base, siteName, keywords, path string, isBot bool) {
-	if isBot {
-		c.Header("Vary", "User-Agent")
-		c.Data(http.StatusNotFound, "text/html; charset=utf-8", []byte(botNotFoundHTML(base, siteName, keywords, path)))
-		return
-	}
+func (h *Handlers) serveNotFound(c *gin.Context, base, siteName, keywords, path string) {
 	embed_static.ServeSPAWithMeta(c, notFoundPageMeta(base, siteName, keywords, path))
 }

--- a/handler/seo_bot.go
+++ b/handler/seo_bot.go
@@ -1,158 +0,0 @@
-package handler
-
-import (
-	"fmt"
-	"html"
-	"strings"
-	"time"
-
-	"git.iioio.com/freefire/jiang13-forum/embed_static"
-	"git.iioio.com/freefire/jiang13-forum/model"
-	"git.iioio.com/freefire/jiang13-forum/service"
-)
-
-// 爬虫专用伪静态 HTML（无 SPA；仅 User-Agent 命中爬虫时返回，避免用户刷新闪屏）
-
-func renderBotHTML(meta *embed_static.SPAPageMeta, bodyInner string) string {
-	if meta == nil {
-		meta = &embed_static.SPAPageMeta{}
-	}
-	ogType := strings.TrimSpace(meta.OGType)
-	if ogType == "" {
-		ogType = "website"
-	}
-	locale := strings.TrimSpace(meta.Locale)
-	if locale == "" {
-		locale = "zh_CN"
-	}
-	var b strings.Builder
-	b.WriteString("<!DOCTYPE html><html lang=\"zh-CN\"><head>")
-	b.WriteString("<meta charset=\"UTF-8\"/>")
-	b.WriteString("<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\"/>")
-	writeEscapedTag(&b, "title", meta.Title)
-	writeEscapedMeta(&b, "name", "description", meta.Description)
-	writeEscapedMeta(&b, "name", "keywords", meta.Keywords)
-	if meta.Robots != "" {
-		writeEscapedMeta(&b, "name", "robots", meta.Robots)
-	}
-	if meta.Canonical != "" {
-		b.WriteString(`<link rel="canonical" href="` + html.EscapeString(meta.Canonical) + `"/>`)
-	}
-	writeEscapedMeta(&b, "property", "og:type", ogType)
-	writeEscapedMeta(&b, "property", "og:site_name", meta.SiteName)
-	writeEscapedMeta(&b, "property", "og:locale", locale)
-	writeEscapedMeta(&b, "property", "og:title", meta.Title)
-	writeEscapedMeta(&b, "property", "og:description", meta.Description)
-	writeEscapedMeta(&b, "property", "og:url", meta.Canonical)
-	writeEscapedMeta(&b, "property", "og:image", meta.OGImage)
-	card := "summary"
-	if strings.TrimSpace(meta.OGImage) != "" {
-		card = "summary_large_image"
-	}
-	writeEscapedMeta(&b, "name", "twitter:card", card)
-	writeEscapedMeta(&b, "name", "twitter:title", meta.Title)
-	writeEscapedMeta(&b, "name", "twitter:description", meta.Description)
-	writeEscapedMeta(&b, "name", "twitter:image", meta.OGImage)
-	if meta.JSONLD != "" {
-		b.WriteString(`<script type="application/ld+json">`)
-		b.WriteString(meta.JSONLD)
-		b.WriteString(`</script>`)
-	}
-	b.WriteString(`<style>
-body{font-family:system-ui,sans-serif;line-height:1.6;max-width:800px;margin:24px auto;padding:0 16px;color:#222}
-a{color:#2d6a4f}img{max-width:100%;height:auto}
-.meta{color:#666;font-size:14px;margin:8px 0 20px}
-.nav{margin:32px 0;font-size:14px}
-</style>`)
-	b.WriteString("</head><body>")
-	b.WriteString(bodyInner)
-	b.WriteString(`<p class="nav"><a href="/">← 返回首页</a></p>`)
-	b.WriteString("</body></html>")
-	return b.String()
-}
-
-func writeEscapedTag(b *strings.Builder, tag, text string) {
-	text = strings.TrimSpace(text)
-	if text == "" {
-		return
-	}
-	b.WriteString("<" + tag + ">" + html.EscapeString(text) + "</" + tag + ">")
-}
-
-func writeEscapedMeta(b *strings.Builder, attr, key, content string) {
-	content = strings.TrimSpace(content)
-	if content == "" {
-		return
-	}
-	b.WriteString("<meta " + attr + "=\"" + html.EscapeString(key) + "\" content=\"" + html.EscapeString(content) + "\"/>")
-}
-
-func (h *Handlers) botBoardHTML(meta *embed_static.SPAPageMeta, board model.Board) string {
-	desc := strings.TrimSpace(board.Description)
-	if desc == "" {
-		desc = meta.Description
-	}
-	body := fmt.Sprintf(`<h1>%s</h1><p class="meta">%s</p>`,
-		html.EscapeString(board.Name),
-		html.EscapeString(desc),
-	)
-	return renderBotHTML(meta, body)
-}
-
-func (h *Handlers) botHomeHTML(meta *embed_static.SPAPageMeta, brand service.SiteBranding) string {
-	name := strings.TrimSpace(brand.Name)
-	if name == "" {
-		name = "姜十三论坛"
-	}
-	intro := brand.MetaDescription()
-	if intro == "" {
-		intro = brand.Slogan
-	}
-	var body strings.Builder
-	body.WriteString("<h1>" + html.EscapeString(name) + "</h1>")
-	if intro != "" {
-		body.WriteString("<p>" + html.EscapeString(intro) + "</p>")
-	}
-	body.WriteString(`<p><a href="/projects">浏览项目</a></p>`)
-	return renderBotHTML(meta, body.String())
-}
-
-func (h *Handlers) botPostHTML(base, siteName, defaultImage, keywords string, post *model.Post) string {
-	meta := attachSiteSEO(h.postPageMeta(base, siteName, defaultImage, post), siteName, keywords)
-	content := service.SanitizePostHTML(service.RedactGatedPostHTML(post.Content))
-	author := service.DisplayName(&post.User)
-	var body strings.Builder
-	body.WriteString("<article>")
-	body.WriteString("<h1>" + html.EscapeString(post.Title) + "</h1>")
-	body.WriteString(`<p class="meta">`)
-	body.WriteString(html.EscapeString(author))
-	body.WriteString(" · ")
-	body.WriteString(html.EscapeString(post.CreatedAt.Local().Format("2006-01-02 15:04")))
-	if post.Board.Name != "" {
-		body.WriteString(" · ")
-		body.WriteString(html.EscapeString(post.Board.Name))
-	}
-	body.WriteString("</p>")
-	body.WriteString(content)
-	body.WriteString("</article>")
-	return renderBotHTML(meta, body.String())
-}
-
-func (h *Handlers) botUserHTML(base, siteName, defaultImage, keywords string, user *model.User) string {
-	meta := attachSiteSEO(h.userPageMeta(base, siteName, defaultImage, user), siteName, keywords)
-	name := service.DisplayName(user)
-	sig := strings.TrimSpace(user.Signature)
-	var body strings.Builder
-	body.WriteString("<h1>" + html.EscapeString(name) + " 的主页</h1>")
-	if sig != "" {
-		body.WriteString("<p>" + html.EscapeString(sig) + "</p>")
-	}
-	body.WriteString(fmt.Sprintf(`<p class="meta">加入于 %s</p>`, html.EscapeString(user.CreatedAt.Local().Format(time.DateOnly))))
-	return renderBotHTML(meta, body.String())
-}
-
-func botNotFoundHTML(base, siteName, keywords, path string) string {
-	meta := notFoundPageMeta(base, siteName, keywords, path)
-	body := `<h1>页面不存在</h1><p>您访问的页面不存在或已删除。</p>`
-	return renderBotHTML(meta, body)
-}
--- a/router/router.go
+++ b/router/router.go
@@ -57,6 +57,9 @@ func Setup(cfg *config.Config) (*gin.Engine, error) {
 		}
 		return b
 	})
+	embed_static.SetSPAFaviconURL(func() string {
+		return settingsSvc.SiteBranding().Favicon
+	})
 	authSvc := service.NewAuthService(cfg.JWTSecret, filter, settingsSvc)
 	userSvc := service.NewUserService(filter, settingsSvc)
 	boardSvc := service.NewBoardService()
@@ -114,9 +117,10 @@ func Setup(cfg *config.Config) (*gin.Engine, error) {
 	// 健康检查（容器 / 负载均衡探活）
 	r.GET("/health", h.APIHealth)

-	// SEO：抓取规则与站点地图
+	// SEO：抓取规则、站点地图、约定 favicon
 	r.GET("/robots.txt", h.RobotsTxt)
 	r.GET("/sitemap.xml", h.SitemapXML)
+	r.GET("/favicon.ico", h.FaviconICO)

 	// OIDC Provider（Gitea 等外部站点 SSO）
 	r.GET("/.well-known/openid-configuration", h.OIDCDiscovery)
--- a/service/crawler.go
+++ b/service/crawler.go
@@ -40,7 +40,7 @@ var seoCrawlerTokens = []string{
 	"oai-searchbot",
 }

-// IsSEOCrawler 是否为需要服务端 HTML 的爬虫 / 预览 bot（动态渲染）
+// IsSEOCrawler 是否为搜索引擎 / 社交预览类爬虫（供访问监控打 is_bot；公开页不再按 UA 分叉 HTML）
 func IsSEOCrawler(userAgent string) bool {
 	ua := strings.ToLower(strings.TrimSpace(userAgent))
 	if ua == "" {