freefire/jiang13-forum
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix: 拆除爬虫专用 HTML，并补齐 /favicon.ico 与 head 图标注入

Browse Source 
人机统一走 SPA/首页 SSR；约定路径与 link rel=icon 便于爬虫识别站点图标。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
姜十三
2026-09-01 07:15:06 +08:00
parent cf7a37eccc
commit 4ca5c17e2e
6 changed files with 50 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
embed_static/embed.go
View File

@@ -14,9 +14,10 @@ import (
var staticFS embed.FS var staticFS embed.FS
var ( var (
spaTitleRe = regexp.MustCompile(`(?s)<title>.*?</title>`) spaTitleRe = regexp.MustCompile(`(?s)<title>.*?</title>`)
spaBrandTitleFn func() string spaBrandTitleFn func() string
spaBrandJSONFn func() []byte // 站点品牌 JSON，注入 window.__J13_BRANDING__ spaBrandJSONFn func() []byte // 站点品牌 JSON，注入 window.__J13_BRANDING__
spaBrandFaviconFn func() string // 站点 Favicon URL，注入 <link rel="icon">
) )
// SetSPADocumentTitle 注册站点标题提供者，ServeSPA 会注入到入口 HTML，避免刷新闪烁 // SetSPADocumentTitle 注册站点标题提供者，ServeSPA 会注入到入口 HTML，避免刷新闪烁
@@ -29,6 +30,11 @@ func SetSPABrandingJSON(fn func() []byte) {
spaBrandJSONFn = fn spaBrandJSONFn = fn
} }
// SetSPAFaviconURL 注册 Favicon URL 提供者，注入到入口 HTML 的 <link rel="icon">
func SetSPAFaviconURL(fn func() string) {
spaBrandFaviconFn = fn
}
// SetupEmbed 配置内嵌资源：React SPA 静态资源 // SetupEmbed 配置内嵌资源：React SPA 静态资源
func SetupEmbed(r *gin.Engine) error { func SetupEmbed(r *gin.Engine) error {
if sub, err := fs.Sub(staticFS, "static/spa/assets"); err == nil { if sub, err := fs.Sub(staticFS, "static/spa/assets"); err == nil {

14
embed_static/spa_meta.go
View File

@@ -62,13 +62,17 @@ func applySPAPageMeta(data []byte, meta *SPAPageMeta) []byte {
data = spaTitleRe.ReplaceAll(data, []byte("<title>"+escaped+"</title>")) data = spaTitleRe.ReplaceAll(data, []byte("<title>"+escaped+"</title>"))
} }
// —— 静态 SEO HTML（meta / OG / JSON-LD），紧跟 </title>，不经 JS —— // —— 静态 SEO HTML（meta / OG / JSON-LD / favicon），紧跟 </title>，不经 JS ——
var seo strings.Builder var seo strings.Builder
writeMeta(&seo, "description", meta.Description) writeMeta(&seo, "description", meta.Description)
writeMeta(&seo, "keywords", meta.Keywords) writeMeta(&seo, "keywords", meta.Keywords)
if canonical := strings.TrimSpace(meta.Canonical); canonical != "" { if canonical := strings.TrimSpace(meta.Canonical); canonical != "" {
seo.WriteString(`<link rel="canonical" href="` + html.EscapeString(canonical) + `"/>`) seo.WriteString(`<link rel="canonical" href="` + html.EscapeString(canonical) + `"/>`)
} }
if favicon := spaFaviconHref(); favicon != "" {
seo.WriteString(`<link rel="icon" href="` + html.EscapeString(favicon) + `"/>`)
seo.WriteString(`<link rel="shortcut icon" href="` + html.EscapeString(favicon) + `"/>`)
}
robots := strings.TrimSpace(meta.Robots) robots := strings.TrimSpace(meta.Robots)
if robots != "" { if robots != "" {
writeMeta(&seo, "robots", robots) writeMeta(&seo, "robots", robots)
@@ -187,3 +191,11 @@ func firstNonEmpty(vals ...string) string {
} }
return "" return ""
} }
// spaFaviconHref 当前站点配置的 Favicon（相对或绝对 URL）
func spaFaviconHref() string {
if spaBrandFaviconFn == nil {
return ""
}
return strings.TrimSpace(spaBrandFaviconFn())
}

58
handler/seo.go
View File

@@ -2,18 +2,16 @@ package handler
import ( import (
"encoding/json" "encoding/json"
"fmt"
"html"
"net/http" "net/http"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/gin-gonic/gin"
"git.iioio.com/freefire/jiang13-forum/embed_static" "git.iioio.com/freefire/jiang13-forum/embed_static"
"git.iioio.com/freefire/jiang13-forum/model" "git.iioio.com/freefire/jiang13-forum/model"
"git.iioio.com/freefire/jiang13-forum/service" "git.iioio.com/freefire/jiang13-forum/service"
"github.com/gin-gonic/gin"
) )
var ( var (
@@ -50,6 +48,18 @@ func (h *Handlers) RobotsTxt(c *gin.Context) {
c.Data(http.StatusOK, "text/plain; charset=utf-8", []byte(b.String())) c.Data(http.StatusOK, "text/plain; charset=utf-8", []byte(b.String()))
} }
// FaviconICO 约定路径 /favicon.ico：有品牌图标则 302 到实际上传 URL，否则 404
func (h *Handlers) FaviconICO(c *gin.Context) {
href := strings.TrimSpace(h.Settings.SiteBranding().Favicon)
if href == "" {
c.Status(http.StatusNotFound)
return
}
// 相对路径保持站内跳转；绝对 URL 也可 Redirect
c.Header("Cache-Control", "public, max-age=86400")
c.Redirect(http.StatusFound, href)
}
// SitemapXML 公开页面站点地图 // SitemapXML 公开页面站点地图
func (h *Handlers) SitemapXML(c *gin.Context) { func (h *Handlers) SitemapXML(c *gin.Context) {
base := h.publicBaseURL(c) base := h.publicBaseURL(c)
@@ -147,9 +157,9 @@ func (h *Handlers) SitemapXML(c *gin.Context) {
c.Data(http.StatusOK, "application/xml; charset=utf-8", []byte(b.String())) c.Data(http.StatusOK, "application/xml; charset=utf-8", []byte(b.String()))
} }
// ServePublicSPA 公开页入口： // ServePublicSPA 公开页入口：人机统一响应，不再按 User-Agent 分叉。
// - 普通用户：干净 SPA + <head> meta（无正文预渲染，避免刷新闪屏） // - 首页 / 板块：完整首屏 SSR（serveFeedDocument）
// - 搜索/社交爬虫：服务端 HTML（动态渲染） // - 其余公开页：干净 SPA + <head> meta
// - 伪静态：按后台配置的后缀做规范 URL，非规范路径 301 // - 伪静态：按后台配置的后缀做规范 URL，非规范路径 301
func (h *Handlers) ServePublicSPA(c *gin.Context) { func (h *Handlers) ServePublicSPA(c *gin.Context) {
path := c.Request.URL.Path path := c.Request.URL.Path
@@ -181,11 +191,6 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
} }
} }
isBot := service.IsSEOCrawler(c.Request.UserAgent())
if isBot {
c.Header("Vary", "User-Agent")
}
// 板块首页（含可选伪静态后缀） // 板块首页（含可选伪静态后缀）
if bm := permalink.MatchBoardPath(path); bm.OK { if bm := permalink.MatchBoardPath(path); bm.OK {
if bm.NeedsCanonicalRedirect(path) { if bm.NeedsCanonicalRedirect(path) {
@@ -194,7 +199,7 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
} }
board, err := h.Board.GetByID(bm.ID) board, err := h.Board.GetByID(bm.ID)
if err != nil { if err != nil {
h.serveNotFound(c, base, siteName, siteKeywords, path, isBot) h.serveNotFound(c, base, siteName, siteKeywords, path)
return return
} }
desc := strings.TrimSpace(board.Description) desc := strings.TrimSpace(board.Description)
@@ -209,7 +214,6 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
OGType: "website", OGType: "website",
OGImage: defaultImage, OGImage: defaultImage,
}, siteName, siteKeywords) }, siteName, siteKeywords)
// 板块首页：真人与爬虫共用完整首屏 HTML
h.serveFeedDocument(c, meta, board.ID) h.serveFeedDocument(c, meta, board.ID)
return return
} }
@@ -222,14 +226,10 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
} }
post, err := h.Post.FindByID(pm.ID) post, err := h.Post.FindByID(pm.ID)
if err != nil || !service.CanViewPost(post, h.currentUserID(c), h.isAdmin(c)) { if err != nil || !service.CanViewPost(post, h.currentUserID(c), h.isAdmin(c)) {
h.serveNotFound(c, base, siteName, siteKeywords, path, isBot) h.serveNotFound(c, base, siteName, siteKeywords, path)
return return
} }
postKeywords := service.JoinSEOKeywords(post.Board.Name, siteKeywords) postKeywords := service.JoinSEOKeywords(post.Board.Name, siteKeywords)
if isBot {
c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(h.botPostHTML(base, siteName, defaultImage, postKeywords, post)))
return
}
embed_static.ServeSPAWithMeta(c, attachSiteSEO(h.postPageMeta(base, siteName, defaultImage, post), siteName, postKeywords)) embed_static.ServeSPAWithMeta(c, attachSiteSEO(h.postPageMeta(base, siteName, defaultImage, post), siteName, postKeywords))
return return
} }
@@ -242,11 +242,7 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
} }
user, err := h.User.GetByID(um.ID) user, err := h.User.GetByID(um.ID)
if err != nil || user.Banned { if err != nil || user.Banned {
h.serveNotFound(c, base, siteName, siteKeywords, path, isBot) h.serveNotFound(c, base, siteName, siteKeywords, path)
return
}
if isBot {
c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(h.botUserHTML(base, siteName, defaultImage, siteKeywords, user)))
return return
} }
embed_static.ServeSPAWithMeta(c, attachSiteSEO(h.userPageMeta(base, siteName, defaultImage, user), siteName, siteKeywords)) embed_static.ServeSPAWithMeta(c, attachSiteSEO(h.userPageMeta(base, siteName, defaultImage, user), siteName, siteKeywords))
@@ -261,7 +257,7 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
} }
page, err := h.SitePage.GetBySlug(pg.Slug, h.isAdmin(c)) page, err := h.SitePage.GetBySlug(pg.Slug, h.isAdmin(c))
if err != nil { if err != nil {
h.serveNotFound(c, base, siteName, siteKeywords, path, isBot) h.serveNotFound(c, base, siteName, siteKeywords, path)
return return
} }
desc := service.ExcerptFromHTML(page.Content, seoDescMax) desc := service.ExcerptFromHTML(page.Content, seoDescMax)
@@ -273,18 +269,13 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
OGType: "article", OGType: "article",
OGImage: defaultImage, OGImage: defaultImage,
}, siteName, siteKeywords) }, siteName, siteKeywords)
if isBot {
body := fmt.Sprintf(`<h1>%s</h1><div>%s</div>`, html.EscapeString(page.Title), page.Content)
c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(renderBotHTML(meta, body)))
return
}
embed_static.ServeSPAWithMeta(c, meta) embed_static.ServeSPAWithMeta(c, meta)
return return
} }
// 未知路径 → 404 // 未知路径 → 404
if !isKnownPublicPath(path) { if !isKnownPublicPath(path) {
h.serveNotFound(c, base, siteName, siteKeywords, path, isBot) h.serveNotFound(c, base, siteName, siteKeywords, path)
return return
} }
@@ -297,12 +288,7 @@ func (h *Handlers) ServePublicSPA(c *gin.Context) {
embed_static.ServeSPAWithMeta(c, meta) embed_static.ServeSPAWithMeta(c, meta)
} }
func (h *Handlers) serveNotFound(c *gin.Context, base, siteName, keywords, path string, isBot bool) { func (h *Handlers) serveNotFound(c *gin.Context, base, siteName, keywords, path string) {
if isBot {
c.Header("Vary", "User-Agent")
c.Data(http.StatusNotFound, "text/html; charset=utf-8", []byte(botNotFoundHTML(base, siteName, keywords, path)))
return
}
embed_static.ServeSPAWithMeta(c, notFoundPageMeta(base, siteName, keywords, path)) embed_static.ServeSPAWithMeta(c, notFoundPageMeta(base, siteName, keywords, path))
} }

158
handler/seo_bot.go
View File

@@ -1,158 +0,0 @@
package handler
import (
"fmt"
"html"
"strings"
"time"
"git.iioio.com/freefire/jiang13-forum/embed_static"
"git.iioio.com/freefire/jiang13-forum/model"
"git.iioio.com/freefire/jiang13-forum/service"
)
// 爬虫专用伪静态 HTML（无 SPA；仅 User-Agent 命中爬虫时返回，避免用户刷新闪屏）
func renderBotHTML(meta *embed_static.SPAPageMeta, bodyInner string) string {
if meta == nil {
meta = &embed_static.SPAPageMeta{}
}
ogType := strings.TrimSpace(meta.OGType)
if ogType == "" {
ogType = "website"
}
locale := strings.TrimSpace(meta.Locale)
if locale == "" {
locale = "zh_CN"
}
var b strings.Builder
b.WriteString("<!DOCTYPE html><html lang=\"zh-CN\"><head>")
b.WriteString("<meta charset=\"UTF-8\"/>")
b.WriteString("<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\"/>")
writeEscapedTag(&b, "title", meta.Title)
writeEscapedMeta(&b, "name", "description", meta.Description)
writeEscapedMeta(&b, "name", "keywords", meta.Keywords)
if meta.Robots != "" {
writeEscapedMeta(&b, "name", "robots", meta.Robots)
}
if meta.Canonical != "" {
b.WriteString(`<link rel="canonical" href="` + html.EscapeString(meta.Canonical) + `"/>`)
}
writeEscapedMeta(&b, "property", "og:type", ogType)
writeEscapedMeta(&b, "property", "og:site_name", meta.SiteName)
writeEscapedMeta(&b, "property", "og:locale", locale)
writeEscapedMeta(&b, "property", "og:title", meta.Title)
writeEscapedMeta(&b, "property", "og:description", meta.Description)
writeEscapedMeta(&b, "property", "og:url", meta.Canonical)
writeEscapedMeta(&b, "property", "og:image", meta.OGImage)
card := "summary"
if strings.TrimSpace(meta.OGImage) != "" {
card = "summary_large_image"
}
writeEscapedMeta(&b, "name", "twitter:card", card)
writeEscapedMeta(&b, "name", "twitter:title", meta.Title)
writeEscapedMeta(&b, "name", "twitter:description", meta.Description)
writeEscapedMeta(&b, "name", "twitter:image", meta.OGImage)
if meta.JSONLD != "" {
b.WriteString(`<script type="application/ld+json">`)
b.WriteString(meta.JSONLD)
b.WriteString(`</script>`)
}
b.WriteString(`<style>
body{font-family:system-ui,sans-serif;line-height:1.6;max-width:800px;margin:24px auto;padding:0 16px;color:#222}
a{color:#2d6a4f}img{max-width:100%;height:auto}
.meta{color:#666;font-size:14px;margin:8px 0 20px}
.nav{margin:32px 0;font-size:14px}
</style>`)
b.WriteString("</head><body>")
b.WriteString(bodyInner)
b.WriteString(`<p class="nav"><a href="/">← 返回首页</a></p>`)
b.WriteString("</body></html>")
return b.String()
}
func writeEscapedTag(b *strings.Builder, tag, text string) {
text = strings.TrimSpace(text)
if text == "" {
return
}
b.WriteString("<" + tag + ">" + html.EscapeString(text) + "</" + tag + ">")
}
func writeEscapedMeta(b *strings.Builder, attr, key, content string) {
content = strings.TrimSpace(content)
if content == "" {
return
}
b.WriteString("<meta " + attr + "=\"" + html.EscapeString(key) + "\" content=\"" + html.EscapeString(content) + "\"/>")
}
func (h *Handlers) botBoardHTML(meta *embed_static.SPAPageMeta, board model.Board) string {
desc := strings.TrimSpace(board.Description)
if desc == "" {
desc = meta.Description
}
body := fmt.Sprintf(`<h1>%s</h1><p class="meta">%s</p>`,
html.EscapeString(board.Name),
html.EscapeString(desc),
)
return renderBotHTML(meta, body)
}
func (h *Handlers) botHomeHTML(meta *embed_static.SPAPageMeta, brand service.SiteBranding) string {
name := strings.TrimSpace(brand.Name)
if name == "" {
name = "姜十三论坛"
}
intro := brand.MetaDescription()
if intro == "" {
intro = brand.Slogan
}
var body strings.Builder
body.WriteString("<h1>" + html.EscapeString(name) + "</h1>")
if intro != "" {
body.WriteString("<p>" + html.EscapeString(intro) + "</p>")
}
body.WriteString(`<p><a href="/projects">浏览项目</a></p>`)
return renderBotHTML(meta, body.String())
}
func (h *Handlers) botPostHTML(base, siteName, defaultImage, keywords string, post *model.Post) string {
meta := attachSiteSEO(h.postPageMeta(base, siteName, defaultImage, post), siteName, keywords)
content := service.SanitizePostHTML(service.RedactGatedPostHTML(post.Content))
author := service.DisplayName(&post.User)
var body strings.Builder
body.WriteString("<article>")
body.WriteString("<h1>" + html.EscapeString(post.Title) + "</h1>")
body.WriteString(`<p class="meta">`)
body.WriteString(html.EscapeString(author))
body.WriteString(" · ")
body.WriteString(html.EscapeString(post.CreatedAt.Local().Format("2006-01-02 15:04")))
if post.Board.Name != "" {
body.WriteString(" · ")
body.WriteString(html.EscapeString(post.Board.Name))
}
body.WriteString("</p>")
body.WriteString(content)
body.WriteString("</article>")
return renderBotHTML(meta, body.String())
}
func (h *Handlers) botUserHTML(base, siteName, defaultImage, keywords string, user *model.User) string {
meta := attachSiteSEO(h.userPageMeta(base, siteName, defaultImage, user), siteName, keywords)
name := service.DisplayName(user)
sig := strings.TrimSpace(user.Signature)
var body strings.Builder
body.WriteString("<h1>" + html.EscapeString(name) + " 的主页</h1>")
if sig != "" {
body.WriteString("<p>" + html.EscapeString(sig) + "</p>")
}
body.WriteString(fmt.Sprintf(`<p class="meta">加入于 %s</p>`, html.EscapeString(user.CreatedAt.Local().Format(time.DateOnly))))
return renderBotHTML(meta, body.String())
}
func botNotFoundHTML(base, siteName, keywords, path string) string {
meta := notFoundPageMeta(base, siteName, keywords, path)
body := `<h1>页面不存在</h1><p>您访问的页面不存在或已删除。</p>`
return renderBotHTML(meta, body)
}

6
router/router.go
View File

@@ -57,6 +57,9 @@ func Setup(cfg *config.Config) (*gin.Engine, error) {
} }
return b return b
}) })
embed_static.SetSPAFaviconURL(func() string {
return settingsSvc.SiteBranding().Favicon
})
authSvc := service.NewAuthService(cfg.JWTSecret, filter, settingsSvc) authSvc := service.NewAuthService(cfg.JWTSecret, filter, settingsSvc)
userSvc := service.NewUserService(filter, settingsSvc) userSvc := service.NewUserService(filter, settingsSvc)
boardSvc := service.NewBoardService() boardSvc := service.NewBoardService()
@@ -114,9 +117,10 @@ func Setup(cfg *config.Config) (*gin.Engine, error) {
// 健康检查（容器 / 负载均衡探活） // 健康检查（容器 / 负载均衡探活）
r.GET("/health", h.APIHealth) r.GET("/health", h.APIHealth)
// SEO：抓取规则与站点地图 // SEO：抓取规则、站点地图、约定 favicon
r.GET("/robots.txt", h.RobotsTxt) r.GET("/robots.txt", h.RobotsTxt)
r.GET("/sitemap.xml", h.SitemapXML) r.GET("/sitemap.xml", h.SitemapXML)
r.GET("/favicon.ico", h.FaviconICO)
// OIDC Provider（Gitea 等外部站点 SSO） // OIDC Provider（Gitea 等外部站点 SSO）
r.GET("/.well-known/openid-configuration", h.OIDCDiscovery) r.GET("/.well-known/openid-configuration", h.OIDCDiscovery)

2
service/crawler.go
View File

@@ -40,7 +40,7 @@ var seoCrawlerTokens = []string{
"oai-searchbot", "oai-searchbot",
} }
// IsSEOCrawler 是否为需要服务端 HTML 的爬虫 / 预览 bot（动态渲染） // IsSEOCrawler 是否为搜索引擎 / 社交预览类爬虫（供访问监控打 is_bot；公开页不再按 UA 分叉 HTML）
func IsSEOCrawler(userAgent string) bool { func IsSEOCrawler(userAgent string) bool {
ua := strings.ToLower(strings.TrimSpace(userAgent)) ua := strings.ToLower(strings.TrimSpace(userAgent))
if ua == "" { if ua == "" {