freefire
429956c594
请求日志按日 JSONL；page_views 不进主库，避免统计数据撑大 jiang13.db。 Co-authored-by: Cursor <cursoragent@cursor.com>
1226 lines
30 KiB
Go
1226 lines
30 KiB
Go
package service
|
||
|
||
import (
|
||
"bufio"
|
||
"encoding/json"
|
||
"fmt"
|
||
"net"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"git.iioio.com/freefire/jiang13-forum/model"
|
||
"gorm.io/gorm"
|
||
)
|
||
|
||
const (
|
||
monitorFlushSize = 64
|
||
monitorFlushInterval = 2 * time.Second
|
||
monitorCleanupEvery = 1 * time.Hour
|
||
monitorQueueMax = 8192
|
||
)
|
||
|
||
// AccessLogLite 中间件入队用的轻量访问日志（不含 Geo）
|
||
type AccessLogLite struct {
|
||
CreatedAt time.Time
|
||
Method string
|
||
Path string
|
||
Status int
|
||
Bytes int64
|
||
DurationMs int
|
||
IP string
|
||
UA string
|
||
Referer string
|
||
CDNCountry string // 仅 CDN 头，非 BIN 查询
|
||
IsBot bool
|
||
}
|
||
|
||
// accessLogJSON 写入 JSONL 的单行结构
|
||
type accessLogJSON struct {
|
||
T string `json:"t"`
|
||
Method string `json:"method"`
|
||
Path string `json:"path"`
|
||
Status int `json:"status"`
|
||
Bytes int64 `json:"bytes"`
|
||
DurationMs int `json:"duration_ms"`
|
||
IP string `json:"ip"`
|
||
UA string `json:"ua"`
|
||
Referer string `json:"referer"`
|
||
Country string `json:"country"`
|
||
Region string `json:"region"`
|
||
RegionISO string `json:"region_iso"`
|
||
City string `json:"city"`
|
||
ASN uint `json:"asn"`
|
||
ASOrg string `json:"as_org"`
|
||
IsBot bool `json:"is_bot"`
|
||
}
|
||
|
||
type monitorDayStats struct {
|
||
dayKey string
|
||
requests int64
|
||
traffic int64
|
||
bots int64
|
||
status4xx int64
|
||
status5xx int64
|
||
uniqueIPs map[string]struct{}
|
||
statusCounts map[int]int64
|
||
}
|
||
|
||
type monitorMinuteBucket struct {
|
||
count int64
|
||
bytes int64
|
||
}
|
||
|
||
// MonitorService 网站监控：JSONL 请求日志 + 独立 monitor.db pageview
|
||
type MonitorService struct {
|
||
settings *ForumSettingsService
|
||
dataDir string
|
||
accessLogDir string
|
||
|
||
mu sync.Mutex
|
||
queue []AccessLogLite
|
||
stopCh chan struct{}
|
||
wg sync.WaitGroup
|
||
|
||
geo *geoIPSuite
|
||
|
||
dayMu sync.RWMutex
|
||
day monitorDayStats
|
||
|
||
rtMu sync.RWMutex
|
||
rtMinute map[string]monitorMinuteBucket // key: 2006-01-02 15:04
|
||
}
|
||
|
||
// NewMonitorService 创建监控服务
|
||
func NewMonitorService(settings *ForumSettingsService, dataDir, _ string) *MonitorService {
|
||
accessDir := filepath.Join(dataDir, "logs", "access")
|
||
_ = os.MkdirAll(accessDir, 0o755)
|
||
now := time.Now()
|
||
dayKey := now.Format("2006-01-02")
|
||
return &MonitorService{
|
||
settings: settings,
|
||
dataDir: dataDir,
|
||
accessLogDir: accessDir,
|
||
queue: make([]AccessLogLite, 0, monitorFlushSize),
|
||
stopCh: make(chan struct{}),
|
||
geo: newGeoIPSuite(dataDir),
|
||
day: monitorDayStats{
|
||
dayKey: dayKey,
|
||
uniqueIPs: map[string]struct{}{},
|
||
statusCounts: map[int]int64{},
|
||
},
|
||
rtMinute: map[string]monitorMinuteBucket{},
|
||
}
|
||
}
|
||
|
||
// StartBackground 启动刷盘与清理协程
|
||
func (m *MonitorService) StartBackground() {
|
||
m.wg.Add(2)
|
||
go m.flushLoop()
|
||
go m.cleanupLoop()
|
||
}
|
||
|
||
// Enabled 采集是否开启
|
||
func (m *MonitorService) Enabled() bool {
|
||
return m != nil && m.settings != nil && m.settings.MonitorEnabled()
|
||
}
|
||
|
||
// Stop 停止后台任务并刷盘
|
||
func (m *MonitorService) Stop() {
|
||
close(m.stopCh)
|
||
m.wg.Wait()
|
||
m.flush()
|
||
if m.geo != nil {
|
||
m.geo.Close()
|
||
}
|
||
}
|
||
|
||
func (m *MonitorService) flushLoop() {
|
||
defer m.wg.Done()
|
||
t := time.NewTicker(monitorFlushInterval)
|
||
defer t.Stop()
|
||
for {
|
||
select {
|
||
case <-m.stopCh:
|
||
return
|
||
case <-t.C:
|
||
m.flush()
|
||
}
|
||
}
|
||
}
|
||
|
||
func (m *MonitorService) cleanupLoop() {
|
||
defer m.wg.Done()
|
||
t := time.NewTicker(monitorCleanupEvery)
|
||
defer t.Stop()
|
||
m.PurgeExpired()
|
||
for {
|
||
select {
|
||
case <-m.stopCh:
|
||
return
|
||
case <-t.C:
|
||
m.PurgeExpired()
|
||
if m.geo != nil {
|
||
m.geo.ReloadIfNeeded()
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Enqueue 缓冲一条访问日志（中间件调用；队列满则丢弃新日志）
|
||
func (m *MonitorService) Enqueue(row AccessLogLite) {
|
||
if m == nil || !m.settings.MonitorEnabled() {
|
||
return
|
||
}
|
||
m.mu.Lock()
|
||
if len(m.queue) >= monitorQueueMax {
|
||
m.mu.Unlock()
|
||
return
|
||
}
|
||
m.queue = append(m.queue, row)
|
||
needFlush := len(m.queue) >= monitorFlushSize
|
||
m.mu.Unlock()
|
||
if needFlush {
|
||
m.flush()
|
||
}
|
||
}
|
||
|
||
func (m *MonitorService) flush() {
|
||
m.mu.Lock()
|
||
if len(m.queue) == 0 {
|
||
m.mu.Unlock()
|
||
return
|
||
}
|
||
batch := m.queue
|
||
m.queue = make([]AccessLogLite, 0, monitorFlushSize)
|
||
m.mu.Unlock()
|
||
|
||
if len(batch) == 0 {
|
||
return
|
||
}
|
||
|
||
byDay := map[string][]accessLogJSON{}
|
||
for _, lite := range batch {
|
||
geo := m.resolveGeoLite(lite.IP, lite.CDNCountry)
|
||
row := accessLogJSON{
|
||
T: lite.CreatedAt.Format(time.RFC3339),
|
||
Method: lite.Method,
|
||
Path: lite.Path,
|
||
Status: lite.Status,
|
||
Bytes: lite.Bytes,
|
||
DurationMs: lite.DurationMs,
|
||
IP: lite.IP,
|
||
UA: lite.UA,
|
||
Referer: lite.Referer,
|
||
Country: geo.Country,
|
||
Region: geo.Region,
|
||
RegionISO: geo.RegionISO,
|
||
City: geo.City,
|
||
ASN: geo.ASN,
|
||
ASOrg: geo.ASOrg,
|
||
IsBot: lite.IsBot,
|
||
}
|
||
day := lite.CreatedAt.Local().Format("2006-01-02")
|
||
byDay[day] = append(byDay[day], row)
|
||
m.updateCounters(lite, geo)
|
||
}
|
||
|
||
for day, rows := range byDay {
|
||
m.appendJSONL(day, rows)
|
||
}
|
||
}
|
||
|
||
func (m *MonitorService) appendJSONL(day string, rows []accessLogJSON) {
|
||
if len(rows) == 0 {
|
||
return
|
||
}
|
||
path := filepath.Join(m.accessLogDir, day+".jsonl")
|
||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer f.Close()
|
||
enc := json.NewEncoder(f)
|
||
for _, row := range rows {
|
||
_ = enc.Encode(row)
|
||
}
|
||
}
|
||
|
||
func (m *MonitorService) ensureDay(now time.Time) {
|
||
key := now.Local().Format("2006-01-02")
|
||
m.dayMu.Lock()
|
||
defer m.dayMu.Unlock()
|
||
if m.day.dayKey == key {
|
||
return
|
||
}
|
||
m.day = monitorDayStats{
|
||
dayKey: key,
|
||
uniqueIPs: map[string]struct{}{},
|
||
statusCounts: map[int]int64{},
|
||
}
|
||
}
|
||
|
||
func (m *MonitorService) updateCounters(lite AccessLogLite, _ GeoInfo) {
|
||
m.ensureDay(lite.CreatedAt)
|
||
minKey := lite.CreatedAt.Local().Truncate(time.Minute).Format("2006-01-02 15:04")
|
||
|
||
m.dayMu.Lock()
|
||
if lite.CreatedAt.Local().Format("2006-01-02") == m.day.dayKey {
|
||
m.day.requests++
|
||
m.day.traffic += lite.Bytes
|
||
if lite.IsBot {
|
||
m.day.bots++
|
||
}
|
||
if lite.Status >= 400 && lite.Status < 500 {
|
||
m.day.status4xx++
|
||
}
|
||
if lite.Status >= 500 {
|
||
m.day.status5xx++
|
||
}
|
||
if ip := strings.TrimSpace(lite.IP); ip != "" {
|
||
m.day.uniqueIPs[ip] = struct{}{}
|
||
}
|
||
m.day.statusCounts[lite.Status]++
|
||
}
|
||
m.dayMu.Unlock()
|
||
|
||
m.rtMu.Lock()
|
||
b := m.rtMinute[minKey]
|
||
b.count++
|
||
b.bytes += lite.Bytes
|
||
m.rtMinute[minKey] = b
|
||
// 清理 2 小时前的分钟桶
|
||
cutoff := time.Now().Add(-2 * time.Hour).Truncate(time.Minute)
|
||
for k := range m.rtMinute {
|
||
t, err := time.ParseInLocation("2006-01-02 15:04", k, time.Local)
|
||
if err != nil || t.Before(cutoff) {
|
||
delete(m.rtMinute, k)
|
||
}
|
||
}
|
||
m.rtMu.Unlock()
|
||
}
|
||
|
||
// pageViewDB 浏览量独立库；未初始化时返回 nil
|
||
func pageViewDB() *gorm.DB {
|
||
return model.MonitorDB
|
||
}
|
||
|
||
// PurgeExpired 按保留天数删除过期 pageview 与 jsonl 请求日志
|
||
func (m *MonitorService) PurgeExpired() {
|
||
if m == nil || m.settings == nil {
|
||
return
|
||
}
|
||
cfg := m.settings.MonitorConfig()
|
||
pvCutoff := time.Now().AddDate(0, 0, -cfg.RetentionDays)
|
||
if db := pageViewDB(); db != nil {
|
||
_ = db.Where("created_at < ?", pvCutoff).Delete(&model.PageView{}).Error
|
||
}
|
||
m.purgeOldJSONL(cfg.AccessLogRetentionDays)
|
||
}
|
||
|
||
func (m *MonitorService) purgeOldJSONL(retentionDays int) {
|
||
if retentionDays < 1 {
|
||
retentionDays = 1
|
||
}
|
||
cutoff := time.Now().AddDate(0, 0, -retentionDays)
|
||
entries, err := os.ReadDir(m.accessLogDir)
|
||
if err != nil {
|
||
return
|
||
}
|
||
for _, e := range entries {
|
||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".jsonl") {
|
||
continue
|
||
}
|
||
base := strings.TrimSuffix(e.Name(), ".jsonl")
|
||
day, err := time.ParseInLocation("2006-01-02", base, time.Local)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if day.Before(cutoff) {
|
||
_ = os.Remove(filepath.Join(m.accessLogDir, e.Name()))
|
||
}
|
||
}
|
||
}
|
||
|
||
// ShouldSkip 是否按排除规则跳过
|
||
func (m *MonitorService) ShouldSkip(path string) bool {
|
||
path = strings.TrimSpace(path)
|
||
if path == "" {
|
||
return true
|
||
}
|
||
lower := strings.ToLower(path)
|
||
if lower == "/api/monitor/pageview" || strings.HasPrefix(lower, "/api/monitor/pageview?") {
|
||
return true
|
||
}
|
||
for _, rule := range m.settings.MonitorConfig().ExcludeRules {
|
||
rule = strings.TrimSpace(rule)
|
||
if rule == "" {
|
||
continue
|
||
}
|
||
r := strings.ToLower(rule)
|
||
if strings.HasPrefix(r, ".") {
|
||
if strings.HasSuffix(lower, r) {
|
||
return true
|
||
}
|
||
continue
|
||
}
|
||
if strings.HasPrefix(lower, r) || lower == strings.TrimSuffix(r, "/") {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// ResolveClientIP 解析客户端 IP（可选信任代理头）
|
||
func (m *MonitorService) ResolveClientIP(r *http.Request, remoteAddr string) string {
|
||
if m.settings.MonitorConfig().TrustProxy {
|
||
for _, h := range []string{"CF-Connecting-IP", "True-Client-IP", "X-Real-IP"} {
|
||
if v := strings.TrimSpace(r.Header.Get(h)); v != "" {
|
||
if ip := firstIP(v); ip != "" {
|
||
return ip
|
||
}
|
||
}
|
||
}
|
||
if v := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); v != "" {
|
||
if ip := firstIP(v); ip != "" {
|
||
return ip
|
||
}
|
||
}
|
||
}
|
||
host, _, err := net.SplitHostPort(remoteAddr)
|
||
if err == nil {
|
||
return host
|
||
}
|
||
return strings.TrimSpace(remoteAddr)
|
||
}
|
||
|
||
func firstIP(v string) string {
|
||
parts := strings.Split(v, ",")
|
||
if len(parts) == 0 {
|
||
return ""
|
||
}
|
||
ip := strings.TrimSpace(parts[0])
|
||
if net.ParseIP(ip) == nil {
|
||
return ""
|
||
}
|
||
return ip
|
||
}
|
||
|
||
func cdnCountryFromRequest(r *http.Request) string {
|
||
if r == nil {
|
||
return ""
|
||
}
|
||
for _, h := range []string{"CF-IPCountry", "CloudFront-Viewer-Country", "X-Country-Code", "X-AppEngine-Country"} {
|
||
if v := normalizeCountryCode(r.Header.Get(h)); v != "" {
|
||
return v
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// ResolveGeo 解析地理与 ASN：CDN 头可补全国家
|
||
func (m *MonitorService) ResolveGeo(r *http.Request, ip string) GeoInfo {
|
||
cdn := cdnCountryFromRequest(r)
|
||
return m.resolveGeoLite(ip, cdn)
|
||
}
|
||
|
||
func (m *MonitorService) resolveGeoLite(ip, cdnCountry string) GeoInfo {
|
||
var out GeoInfo
|
||
if m != nil && m.geo != nil {
|
||
out = m.geo.Lookup(ip)
|
||
}
|
||
if out.Country == "" && cdnCountry != "" {
|
||
out.Country = normalizeCountryCode(cdnCountry)
|
||
ApplyGeoZh(&out)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func normalizeCountryCode(v string) string {
|
||
v = strings.ToUpper(strings.TrimSpace(v))
|
||
if len(v) != 2 || v == "XX" || v == "T1" {
|
||
return ""
|
||
}
|
||
for _, c := range v {
|
||
if c < 'A' || c > 'Z' {
|
||
return ""
|
||
}
|
||
}
|
||
return v
|
||
}
|
||
|
||
// EnrichGeoMeta 填充设置中的 Geo 库与访问日志目录状态
|
||
func (m *MonitorService) EnrichGeoMeta(cfg *MonitorConfig) {
|
||
if cfg == nil || m == nil {
|
||
return
|
||
}
|
||
cfg.AccessLogDir = m.accessLogDir
|
||
cfg.DefaultExcludeRules = DefaultMonitorExcludeRules()
|
||
if m.settings != nil {
|
||
cfg.AccessLogRetentionDays = m.settings.MonitorConfig().AccessLogRetentionDays
|
||
}
|
||
v4, v6, asn, country := "", "", "", ""
|
||
if m.geo != nil {
|
||
v4, v6, asn, country = m.geo.Paths()
|
||
cfg.IP2LocationV4Available = m.geo.BINV4Available()
|
||
cfg.IP2LocationV6Available = m.geo.BINV6Available()
|
||
cfg.GeoIPASNAvailable = m.geo.ASNAvailable()
|
||
cfg.GeoIPCountryAvailable = m.geo.CountryAvailable()
|
||
cfg.GeoIPAvailable = m.geo.AnyAvailable()
|
||
}
|
||
cfg.IP2LocationV4Path = v4
|
||
cfg.IP2LocationV6Path = v6
|
||
cfg.GeoIPASNPath = asn
|
||
cfg.GeoIPCountryPath = country
|
||
}
|
||
|
||
// MonitorOverview 今日概览
|
||
type MonitorOverview struct {
|
||
Enabled bool `json:"enabled"`
|
||
Pageviews int64 `json:"pageviews"`
|
||
Visitors int64 `json:"visitors"`
|
||
UniqueIPs int64 `json:"unique_ips"`
|
||
Traffic int64 `json:"traffic"`
|
||
Bots int64 `json:"bots"`
|
||
Requests int64 `json:"requests"`
|
||
Status4xx int64 `json:"status_4xx"`
|
||
Status5xx int64 `json:"status_5xx"`
|
||
}
|
||
|
||
// MonitorGeoItem 国家排行
|
||
type MonitorGeoItem struct {
|
||
Country string `json:"country"`
|
||
Count int64 `json:"count"`
|
||
}
|
||
|
||
// MonitorRegionItem 省/州排行
|
||
type MonitorRegionItem struct {
|
||
Country string `json:"country"`
|
||
Region string `json:"region"`
|
||
RegionISO string `json:"region_iso"`
|
||
Count int64 `json:"count"`
|
||
}
|
||
|
||
// MonitorCityItem 城市排行
|
||
type MonitorCityItem struct {
|
||
Country string `json:"country"`
|
||
Region string `json:"region"`
|
||
City string `json:"city"`
|
||
Count int64 `json:"count"`
|
||
}
|
||
|
||
// MonitorASNItem 运营商（ASN）排行
|
||
type MonitorASNItem struct {
|
||
ASN uint `json:"asn"`
|
||
ASOrg string `json:"as_org"`
|
||
Count int64 `json:"count"`
|
||
}
|
||
|
||
// MonitorGeoResult 地理分布
|
||
type MonitorGeoResult struct {
|
||
Range string `json:"range"`
|
||
Countries []MonitorGeoItem `json:"countries"`
|
||
Regions []MonitorRegionItem `json:"regions"`
|
||
Cities []MonitorCityItem `json:"cities"`
|
||
ASNs []MonitorASNItem `json:"asns"`
|
||
HasData bool `json:"has_data"`
|
||
}
|
||
|
||
// MonitorStatItem 维度排行项
|
||
type MonitorStatItem struct {
|
||
Key string `json:"key"`
|
||
Count int64 `json:"count"`
|
||
}
|
||
|
||
// MonitorRealtime 实时指标
|
||
type MonitorRealtime struct {
|
||
Enabled bool `json:"enabled"`
|
||
Requests1m int64 `json:"requests_1m"`
|
||
Traffic1m int64 `json:"traffic_1m"`
|
||
HourlySeries []MonitorRealtimePoint `json:"hourly_series"`
|
||
}
|
||
|
||
// MonitorRealtimePoint 近 1 小时分钟点
|
||
type MonitorRealtimePoint struct {
|
||
Minute string `json:"minute"`
|
||
Count int64 `json:"count"`
|
||
Bytes int64 `json:"bytes"`
|
||
}
|
||
|
||
// MonitorLogItem 请求日志行
|
||
type MonitorLogItem struct {
|
||
ID uint `json:"id"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
Method string `json:"method"`
|
||
Path string `json:"path"`
|
||
Status int `json:"status"`
|
||
Bytes int64 `json:"bytes"`
|
||
DurationMs int `json:"duration_ms"`
|
||
IP string `json:"ip"`
|
||
UA string `json:"ua"`
|
||
Referer string `json:"referer"`
|
||
Country string `json:"country"`
|
||
Region string `json:"region"`
|
||
City string `json:"city"`
|
||
ASN uint `json:"asn"`
|
||
ASOrg string `json:"as_org"`
|
||
IsBot bool `json:"is_bot"`
|
||
}
|
||
|
||
func startOfLocalDay(t time.Time) time.Time {
|
||
y, m, d := t.Local().Date()
|
||
return time.Date(y, m, d, 0, 0, 0, 0, t.Location())
|
||
}
|
||
|
||
// DashboardTraffic 仪表盘流量摘要（page_views，非 bot）
|
||
type DashboardTraffic struct {
|
||
Enabled bool `json:"enabled"`
|
||
TodayPV int64 `json:"today_pv"`
|
||
TodayUV int64 `json:"today_uv"`
|
||
YesterdayPV int64 `json:"yesterday_pv"`
|
||
TotalPV int64 `json:"total_pv"`
|
||
}
|
||
|
||
// DashboardTraffic 聚合今日/昨日/累计浏览量
|
||
func (m *MonitorService) DashboardTraffic() DashboardTraffic {
|
||
out := DashboardTraffic{Enabled: m != nil && m.settings != nil && m.settings.MonitorEnabled()}
|
||
if m == nil {
|
||
return out
|
||
}
|
||
db := pageViewDB()
|
||
if db == nil {
|
||
return out
|
||
}
|
||
now := time.Now()
|
||
today := startOfLocalDay(now)
|
||
yesterday := today.AddDate(0, 0, -1)
|
||
_ = db.Model(&model.PageView{}).Where("created_at >= ? AND is_bot = ?", today, false).Count(&out.TodayPV).Error
|
||
_ = db.Model(&model.PageView{}).Where("created_at >= ? AND is_bot = ? AND ip <> ''", today, false).
|
||
Distinct("ip").Count(&out.TodayUV).Error
|
||
_ = db.Model(&model.PageView{}).
|
||
Where("created_at >= ? AND created_at < ? AND is_bot = ?", yesterday, today, false).
|
||
Count(&out.YesterdayPV).Error
|
||
_ = db.Model(&model.PageView{}).Where("is_bot = ?", false).Count(&out.TotalPV).Error
|
||
return out
|
||
}
|
||
|
||
// PageViewInput 前台信标入参
|
||
type PageViewInput struct {
|
||
Path string `json:"path"`
|
||
Referrer string `json:"referrer"`
|
||
}
|
||
|
||
// NormalizePageViewPath 校验并规范化前端路径
|
||
func NormalizePageViewPath(raw string) (string, bool) {
|
||
raw = strings.TrimSpace(raw)
|
||
if raw == "" || !strings.HasPrefix(raw, "/") {
|
||
return "", false
|
||
}
|
||
if strings.Contains(raw, "://") || strings.Contains(raw, "\\") {
|
||
return "", false
|
||
}
|
||
if i := strings.IndexByte(raw, '#'); i >= 0 {
|
||
raw = raw[:i]
|
||
}
|
||
if len(raw) > 512 {
|
||
raw = raw[:512]
|
||
}
|
||
pathOnly := raw
|
||
if i := strings.IndexByte(raw, '?'); i >= 0 {
|
||
pathOnly = raw[:i]
|
||
}
|
||
if shouldIgnorePageViewPath(pathOnly) {
|
||
return "", false
|
||
}
|
||
return raw, true
|
||
}
|
||
|
||
func shouldIgnorePageViewPath(path string) bool {
|
||
lower := strings.ToLower(path)
|
||
prefixes := []string{
|
||
"/admin", "/login", "/register", "/forgot-password",
|
||
"/oauth", "/api", "/health", "/uploads", "/media",
|
||
}
|
||
for _, p := range prefixes {
|
||
if lower == p || strings.HasPrefix(lower, p+"/") {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// RecordPageView 写入一条 SPA 浏览记录
|
||
func (m *MonitorService) RecordPageView(r *http.Request, remoteAddr string, in PageViewInput) error {
|
||
if m == nil || !m.settings.MonitorEnabled() {
|
||
return nil
|
||
}
|
||
db := pageViewDB()
|
||
if db == nil {
|
||
return nil
|
||
}
|
||
path, ok := NormalizePageViewPath(in.Path)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
ref := strings.TrimSpace(in.Referrer)
|
||
if strings.Contains(ref, "://") {
|
||
if len(ref) > 512 {
|
||
ref = ref[:512]
|
||
}
|
||
} else if ref != "" && !strings.HasPrefix(ref, "/") {
|
||
ref = ""
|
||
}
|
||
if len(ref) > 512 {
|
||
ref = ref[:512]
|
||
}
|
||
ip := m.ResolveClientIP(r, remoteAddr)
|
||
ua := r.UserAgent()
|
||
if len(ua) > 512 {
|
||
ua = ua[:512]
|
||
}
|
||
geo := m.ResolveGeo(r, ip)
|
||
row := model.PageView{
|
||
CreatedAt: time.Now(),
|
||
Path: path,
|
||
Referrer: ref,
|
||
IP: ip,
|
||
UA: ua,
|
||
Country: geo.Country,
|
||
Region: geo.Region,
|
||
RegionISO: geo.RegionISO,
|
||
City: geo.City,
|
||
ASN: geo.ASN,
|
||
ASOrg: geo.ASOrg,
|
||
IsBot: IsSEOCrawler(ua) || isGenericBot(ua),
|
||
}
|
||
return db.Create(&row).Error
|
||
}
|
||
|
||
func parseMonitorRange(rangeKey string) (time.Time, string) {
|
||
now := time.Now()
|
||
switch strings.ToLower(strings.TrimSpace(rangeKey)) {
|
||
case "7d":
|
||
return now.AddDate(0, 0, -7), "7d"
|
||
case "90d":
|
||
return now.AddDate(0, 0, -90), "90d"
|
||
case "1d", "today":
|
||
return startOfLocalDay(now), "1d"
|
||
default:
|
||
return now.AddDate(0, 0, -30), "30d"
|
||
}
|
||
}
|
||
|
||
// OverviewToday 今日指标（请求类来自内存 + JSONL 口径；浏览量来自 page_views）
|
||
func (m *MonitorService) OverviewToday() MonitorOverview {
|
||
out := MonitorOverview{Enabled: m.settings.MonitorEnabled()}
|
||
m.ensureDay(time.Now())
|
||
|
||
m.dayMu.RLock()
|
||
if m.day.dayKey == time.Now().Local().Format("2006-01-02") {
|
||
out.Requests = m.day.requests
|
||
out.Traffic = m.day.traffic
|
||
out.Bots = m.day.bots
|
||
out.Status4xx = m.day.status4xx
|
||
out.Status5xx = m.day.status5xx
|
||
out.UniqueIPs = int64(len(m.day.uniqueIPs))
|
||
}
|
||
m.dayMu.RUnlock()
|
||
|
||
db := pageViewDB()
|
||
if db == nil {
|
||
return out
|
||
}
|
||
start := startOfLocalDay(time.Now())
|
||
_ = db.Model(&model.PageView{}).Where("created_at >= ? AND is_bot = ?", start, false).Count(&out.Pageviews).Error
|
||
_ = db.Model(&model.PageView{}).Where("created_at >= ? AND is_bot = ? AND ip <> ''", start, false).
|
||
Distinct("ip").Count(&out.Visitors).Error
|
||
return out
|
||
}
|
||
|
||
// GeoStats 地理分布（基于 pageview）
|
||
func (m *MonitorService) GeoStats(rangeKey string) MonitorGeoResult {
|
||
since, rk := parseMonitorRange(rangeKey)
|
||
out := MonitorGeoResult{
|
||
Range: rk,
|
||
Countries: []MonitorGeoItem{},
|
||
Regions: []MonitorRegionItem{},
|
||
Cities: []MonitorCityItem{},
|
||
ASNs: []MonitorASNItem{},
|
||
}
|
||
db := pageViewDB()
|
||
if db == nil {
|
||
return out
|
||
}
|
||
|
||
type countryRow struct {
|
||
Country string
|
||
Count int64
|
||
}
|
||
var countries []countryRow
|
||
_ = db.Model(&model.PageView{}).
|
||
Select("country, COUNT(*) as count").
|
||
Where("created_at >= ? AND country <> '' AND is_bot = ?", since, false).
|
||
Group("country").
|
||
Order("count DESC").
|
||
Limit(50).
|
||
Scan(&countries).Error
|
||
for _, r := range countries {
|
||
out.Countries = append(out.Countries, MonitorGeoItem{Country: r.Country, Count: r.Count})
|
||
}
|
||
|
||
type regionRow struct {
|
||
Country string
|
||
Region string
|
||
RegionISO string
|
||
Count int64
|
||
}
|
||
var regions []regionRow
|
||
_ = db.Model(&model.PageView{}).
|
||
Select("country, region, region_iso, COUNT(*) as count").
|
||
Where("created_at >= ? AND country <> '' AND (region <> '' OR region_iso <> '') AND is_bot = ?", since, false).
|
||
Group("country, region, region_iso").
|
||
Order("count DESC").
|
||
Limit(80).
|
||
Scan(®ions).Error
|
||
for _, r := range regions {
|
||
out.Regions = append(out.Regions, MonitorRegionItem{
|
||
Country: r.Country, Region: r.Region, RegionISO: r.RegionISO, Count: r.Count,
|
||
})
|
||
}
|
||
|
||
type cityRow struct {
|
||
Country string
|
||
Region string
|
||
City string
|
||
Count int64
|
||
}
|
||
var cities []cityRow
|
||
_ = db.Model(&model.PageView{}).
|
||
Select("country, region, city, COUNT(*) as count").
|
||
Where("created_at >= ? AND city <> '' AND is_bot = ?", since, false).
|
||
Group("country, region, city").
|
||
Order("count DESC").
|
||
Limit(50).
|
||
Scan(&cities).Error
|
||
for _, r := range cities {
|
||
out.Cities = append(out.Cities, MonitorCityItem{
|
||
Country: r.Country, Region: r.Region, City: r.City, Count: r.Count,
|
||
})
|
||
}
|
||
|
||
type asnRow struct {
|
||
ASN uint
|
||
ASOrg string
|
||
Count int64
|
||
}
|
||
var asns []asnRow
|
||
_ = db.Model(&model.PageView{}).
|
||
Select("asn, as_org, COUNT(*) as count").
|
||
Where("created_at >= ? AND asn > 0 AND is_bot = ?", since, false).
|
||
Group("asn, as_org").
|
||
Order("count DESC").
|
||
Limit(50).
|
||
Scan(&asns).Error
|
||
for _, r := range asns {
|
||
out.ASNs = append(out.ASNs, MonitorASNItem{ASN: r.ASN, ASOrg: r.ASOrg, Count: r.Count})
|
||
}
|
||
|
||
out.HasData = len(out.Countries) > 0 || len(out.Regions) > 0 || len(out.Cities) > 0 || len(out.ASNs) > 0
|
||
return out
|
||
}
|
||
|
||
// DimStats 维度排行
|
||
func (m *MonitorService) DimStats(dim, rangeKey string) []MonitorStatItem {
|
||
since, rk := parseMonitorRange(rangeKey)
|
||
dim = strings.ToLower(strings.TrimSpace(dim))
|
||
out := []MonitorStatItem{}
|
||
db := pageViewDB()
|
||
|
||
switch dim {
|
||
case "url", "path":
|
||
if db == nil {
|
||
return out
|
||
}
|
||
type row struct {
|
||
Path string
|
||
Count int64
|
||
}
|
||
var rows []row
|
||
_ = db.Model(&model.PageView{}).
|
||
Select("path, COUNT(*) as count").
|
||
Where("created_at >= ?", since).
|
||
Group("path").Order("count DESC").Limit(50).Scan(&rows).Error
|
||
for _, r := range rows {
|
||
out = append(out, MonitorStatItem{Key: r.Path, Count: r.Count})
|
||
}
|
||
case "referer", "referrer":
|
||
if db == nil {
|
||
return out
|
||
}
|
||
type row struct {
|
||
Referrer string
|
||
Count int64
|
||
}
|
||
var rows []row
|
||
_ = db.Model(&model.PageView{}).
|
||
Select("CASE WHEN referrer = '' THEN '(直接访问)' ELSE referrer END as referrer, COUNT(*) as count").
|
||
Where("created_at >= ?", since).
|
||
Group("referrer").Order("count DESC").Limit(50).Scan(&rows).Error
|
||
for _, r := range rows {
|
||
out = append(out, MonitorStatItem{Key: r.Referrer, Count: r.Count})
|
||
}
|
||
case "status":
|
||
if rk == "1d" {
|
||
m.ensureDay(time.Now())
|
||
m.dayMu.RLock()
|
||
if m.day.dayKey == time.Now().Local().Format("2006-01-02") {
|
||
type kv struct {
|
||
status int
|
||
count int64
|
||
}
|
||
list := make([]kv, 0, len(m.day.statusCounts))
|
||
for st, c := range m.day.statusCounts {
|
||
list = append(list, kv{st, c})
|
||
}
|
||
sort.Slice(list, func(i, j int) bool { return list[i].count > list[j].count })
|
||
n := len(list)
|
||
if n > 50 {
|
||
n = 50
|
||
}
|
||
for i := 0; i < n; i++ {
|
||
out = append(out, MonitorStatItem{Key: fmt.Sprintf("%d", list[i].status), Count: list[i].count})
|
||
}
|
||
}
|
||
m.dayMu.RUnlock()
|
||
}
|
||
if len(out) == 0 {
|
||
counts := m.scanJSONLStatusCounts(since)
|
||
type kv struct {
|
||
k string
|
||
c int64
|
||
}
|
||
list := make([]kv, 0, len(counts))
|
||
for st, c := range counts {
|
||
list = append(list, kv{fmt.Sprintf("%d", st), c})
|
||
}
|
||
sort.Slice(list, func(i, j int) bool { return list[i].c > list[j].c })
|
||
n := len(list)
|
||
if n > 50 {
|
||
n = 50
|
||
}
|
||
for i := 0; i < n; i++ {
|
||
out = append(out, MonitorStatItem{Key: list[i].k, Count: list[i].c})
|
||
}
|
||
}
|
||
case "browser", "os", "device":
|
||
if db == nil {
|
||
return out
|
||
}
|
||
var uas []string
|
||
_ = db.Model(&model.PageView{}).
|
||
Where("created_at >= ?", since).
|
||
Limit(20000).
|
||
Pluck("ua", &uas).Error
|
||
counts := map[string]int64{}
|
||
for _, ua := range uas {
|
||
key := classifyUA(ua, dim)
|
||
counts[key]++
|
||
}
|
||
type kv struct {
|
||
k string
|
||
c int64
|
||
}
|
||
list := make([]kv, 0, len(counts))
|
||
for k, c := range counts {
|
||
list = append(list, kv{k, c})
|
||
}
|
||
sort.Slice(list, func(i, j int) bool { return list[i].c > list[j].c })
|
||
n := len(list)
|
||
if n > 50 {
|
||
n = 50
|
||
}
|
||
for i := 0; i < n; i++ {
|
||
out = append(out, MonitorStatItem{Key: list[i].k, Count: list[i].c})
|
||
}
|
||
default:
|
||
return m.DimStats("url", rangeKey)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (m *MonitorService) scanJSONLStatusCounts(since time.Time) map[int]int64 {
|
||
counts := map[int]int64{}
|
||
for _, path := range m.listJSONLFilesSince(since) {
|
||
rows, _ := m.readJSONLFile(path)
|
||
for _, row := range rows {
|
||
t, err := time.Parse(time.RFC3339, row.T)
|
||
if err != nil || t.Before(since) {
|
||
continue
|
||
}
|
||
counts[row.Status]++
|
||
}
|
||
}
|
||
return counts
|
||
}
|
||
|
||
func classifyUA(ua, dim string) string {
|
||
l := strings.ToLower(ua)
|
||
if l == "" {
|
||
return "未知"
|
||
}
|
||
switch dim {
|
||
case "browser":
|
||
switch {
|
||
case strings.Contains(l, "edg/"):
|
||
return "Edge"
|
||
case strings.Contains(l, "chrome") && !strings.Contains(l, "edg"):
|
||
return "Chrome"
|
||
case strings.Contains(l, "firefox"):
|
||
return "Firefox"
|
||
case strings.Contains(l, "safari") && !strings.Contains(l, "chrome"):
|
||
return "Safari"
|
||
case strings.Contains(l, "msie") || strings.Contains(l, "trident"):
|
||
return "IE"
|
||
default:
|
||
return "其他"
|
||
}
|
||
case "os":
|
||
switch {
|
||
case strings.Contains(l, "windows"):
|
||
return "Windows"
|
||
case strings.Contains(l, "android"):
|
||
return "Android"
|
||
case strings.Contains(l, "iphone") || strings.Contains(l, "ipad") || strings.Contains(l, "ios"):
|
||
return "iOS"
|
||
case strings.Contains(l, "mac os") || strings.Contains(l, "macintosh"):
|
||
return "macOS"
|
||
case strings.Contains(l, "linux"):
|
||
return "Linux"
|
||
default:
|
||
return "其他"
|
||
}
|
||
default:
|
||
switch {
|
||
case strings.Contains(l, "mobile") || strings.Contains(l, "android") || strings.Contains(l, "iphone"):
|
||
return "Mobile"
|
||
case strings.Contains(l, "ipad") || strings.Contains(l, "tablet"):
|
||
return "Tablet"
|
||
default:
|
||
return "Desktop"
|
||
}
|
||
}
|
||
}
|
||
|
||
// Realtime 近 1 分钟 + 近 1 小时序列（内存环）
|
||
func (m *MonitorService) Realtime() MonitorRealtime {
|
||
out := MonitorRealtime{
|
||
Enabled: m.settings.MonitorEnabled(),
|
||
HourlySeries: make([]MonitorRealtimePoint, 0, 60),
|
||
}
|
||
now := time.Now()
|
||
since1m := now.Add(-1 * time.Minute).Truncate(time.Minute)
|
||
|
||
m.rtMu.RLock()
|
||
for k, b := range m.rtMinute {
|
||
t, err := time.ParseInLocation("2006-01-02 15:04", k, time.Local)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if !t.Before(since1m) {
|
||
out.Requests1m += b.count
|
||
out.Traffic1m += b.bytes
|
||
}
|
||
}
|
||
byMin := map[string]monitorMinuteBucket{}
|
||
for k, b := range m.rtMinute {
|
||
byMin[k] = b
|
||
}
|
||
m.rtMu.RUnlock()
|
||
|
||
for i := 59; i >= 0; i-- {
|
||
t := now.Add(-time.Duration(i) * time.Minute).Truncate(time.Minute)
|
||
key := t.Format("2006-01-02 15:04")
|
||
pt := MonitorRealtimePoint{Minute: t.Format("15:04")}
|
||
if r, ok := byMin[key]; ok {
|
||
pt.Count = r.count
|
||
pt.Bytes = r.bytes
|
||
}
|
||
out.HourlySeries = append(out.HourlySeries, pt)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ListLogs 从 JSONL 分页筛选请求日志
|
||
func (m *MonitorService) ListLogs(page, size int, method, path, status, ip string) (items []MonitorLogItem, total int64) {
|
||
if page < 1 {
|
||
page = 1
|
||
}
|
||
if size < 1 {
|
||
size = 20
|
||
}
|
||
if size > 100 {
|
||
size = 100
|
||
}
|
||
retention := m.settings.MonitorConfig().AccessLogRetentionDays
|
||
since := time.Now().AddDate(0, 0, -retention)
|
||
|
||
var all []accessLogJSON
|
||
for _, fp := range m.listJSONLFilesSince(since) {
|
||
rows, err := m.readJSONLFile(fp)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
all = append(all, rows...)
|
||
}
|
||
// 新到旧
|
||
sort.Slice(all, func(i, j int) bool {
|
||
return all[i].T > all[j].T
|
||
})
|
||
|
||
method = strings.TrimSpace(strings.ToUpper(method))
|
||
path = strings.TrimSpace(path)
|
||
status = strings.TrimSpace(status)
|
||
ip = strings.TrimSpace(ip)
|
||
|
||
filtered := make([]accessLogJSON, 0, len(all))
|
||
for _, row := range all {
|
||
if method != "" && strings.ToUpper(row.Method) != method {
|
||
continue
|
||
}
|
||
if path != "" && !strings.Contains(row.Path, path) {
|
||
continue
|
||
}
|
||
if status != "" && !strings.HasPrefix(fmt.Sprintf("%d", row.Status), status) {
|
||
continue
|
||
}
|
||
if ip != "" && !strings.Contains(row.IP, ip) {
|
||
continue
|
||
}
|
||
filtered = append(filtered, row)
|
||
}
|
||
|
||
total = int64(len(filtered))
|
||
start := (page - 1) * size
|
||
if start >= len(filtered) {
|
||
return []MonitorLogItem{}, total
|
||
}
|
||
end := start + size
|
||
if end > len(filtered) {
|
||
end = len(filtered)
|
||
}
|
||
items = make([]MonitorLogItem, 0, end-start)
|
||
for i, row := range filtered[start:end] {
|
||
created, _ := time.Parse(time.RFC3339, row.T)
|
||
items = append(items, MonitorLogItem{
|
||
ID: uint(start + i + 1),
|
||
CreatedAt: created,
|
||
Method: row.Method,
|
||
Path: row.Path,
|
||
Status: row.Status,
|
||
Bytes: row.Bytes,
|
||
DurationMs: row.DurationMs,
|
||
IP: row.IP,
|
||
UA: row.UA,
|
||
Referer: row.Referer,
|
||
Country: row.Country,
|
||
Region: row.Region,
|
||
City: row.City,
|
||
ASN: row.ASN,
|
||
ASOrg: row.ASOrg,
|
||
IsBot: row.IsBot,
|
||
})
|
||
}
|
||
return items, total
|
||
}
|
||
|
||
func (m *MonitorService) listJSONLFilesSince(since time.Time) []string {
|
||
entries, err := os.ReadDir(m.accessLogDir)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
var files []string
|
||
for _, e := range entries {
|
||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".jsonl") {
|
||
continue
|
||
}
|
||
base := strings.TrimSuffix(e.Name(), ".jsonl")
|
||
day, err := time.ParseInLocation("2006-01-02", base, time.Local)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if day.Before(startOfLocalDay(since)) {
|
||
continue
|
||
}
|
||
files = append(files, filepath.Join(m.accessLogDir, e.Name()))
|
||
}
|
||
sort.Strings(files)
|
||
return files
|
||
}
|
||
|
||
func (m *MonitorService) readJSONLFile(path string) ([]accessLogJSON, error) {
|
||
f, err := os.Open(path)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer f.Close()
|
||
var rows []accessLogJSON
|
||
sc := bufio.NewScanner(f)
|
||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||
for sc.Scan() {
|
||
line := strings.TrimSpace(sc.Text())
|
||
if line == "" {
|
||
continue
|
||
}
|
||
var row accessLogJSON
|
||
if err := json.Unmarshal([]byte(line), &row); err != nil {
|
||
continue
|
||
}
|
||
rows = append(rows, row)
|
||
}
|
||
return rows, sc.Err()
|
||
}
|
||
|
||
// BuildAccessLog 从请求构造轻量日志（热路径不查 Geo）
|
||
func (m *MonitorService) BuildAccessLog(r *http.Request, remoteAddr string, status int, bytes int64, durationMs int) AccessLogLite {
|
||
ip := m.ResolveClientIP(r, remoteAddr)
|
||
ua := r.UserAgent()
|
||
if len(ua) > 512 {
|
||
ua = ua[:512]
|
||
}
|
||
ref := r.Referer()
|
||
if len(ref) > 512 {
|
||
ref = ref[:512]
|
||
}
|
||
path := r.URL.Path
|
||
if len(path) > 512 {
|
||
path = path[:512]
|
||
}
|
||
return AccessLogLite{
|
||
CreatedAt: time.Now(),
|
||
Method: r.Method,
|
||
Path: path,
|
||
Status: status,
|
||
Bytes: bytes,
|
||
DurationMs: durationMs,
|
||
IP: ip,
|
||
UA: ua,
|
||
Referer: ref,
|
||
CDNCountry: cdnCountryFromRequest(r),
|
||
IsBot: IsSEOCrawler(ua) || isGenericBot(ua),
|
||
}
|
||
}
|
||
|
||
func isGenericBot(ua string) bool {
|
||
l := strings.ToLower(ua)
|
||
for _, t := range []string{"bot", "spider", "crawl", "slurp", "curl/", "wget/", "python-requests", "go-http-client"} {
|
||
if strings.Contains(l, t) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|