4 Commits

Author SHA1 Message Date
45e6bf653f 增加dockerfile配置 2026-04-09 16:10:11 +08:00
qhd
8eb992e658 feat: 添加 Meilisearch 排名分数支持 2026-04-09 13:58:36 +08:00
qhd
5022e9115d feat: 添加分词工具支持并更新字段命名 2026-04-08 14:21:12 +08:00
qhd
7acee87023 feat: 支持多数据库配置与PGVector检索 2026-04-03 17:59:04 +08:00
8 changed files with 152 additions and 3 deletions

16
config/config.go Normal file
View File

@@ -0,0 +1,16 @@
package config
import (
"fmt"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gcfg"
"github.com/gogf/gf/v2/os/genv"
)
func init() {
env := genv.Get("APP_ENV", "").String()
if env != "" {
g.Cfg().GetAdapter().(*gcfg.AdapterFile).SetFileName(fmt.Sprintf("config-%s.yml", env))
}
}

View File

@@ -414,6 +414,7 @@ var (
type Gfdb interface { type Gfdb interface {
Exec(ctx context.Context, sql string, args ...any) (sql.Result, error) Exec(ctx context.Context, sql string, args ...any) (sql.Result, error)
GetAll(ctx context.Context, sql string, args ...any) (gdb.Result, error)
Model(ctx context.Context, tableNameOrStruct ...any) *model Model(ctx context.Context, tableNameOrStruct ...any) *model
Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) error Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) error
} }
@@ -441,7 +442,6 @@ type modelCache struct {
func checkSchemaConfig(ctx context.Context) (uint64, bool) { func checkSchemaConfig(ctx context.Context) (uint64, bool) {
user, err := utils.GetUserInfo(ctx) user, err := utils.GetUserInfo(ctx)
if err != nil { if err != nil {
glog.Errorf(ctx, "[DB] GetUserInfo error: %v", err)
return 0, false return 0, false
} }
var schema = fmt.Sprintf("%s%v", schemaPrefix, user.TenantId) var schema = fmt.Sprintf("%s%v", schemaPrefix, user.TenantId)

View File

@@ -118,7 +118,7 @@ func (m *meilisearchDB) ensureIndexExists(client ms.ServiceManager, indexName st
return err return err
} }
requiredFilterable := []string{"tenantId", "isDeleted", "datasetId", "creator", "updater"} requiredFilterable := []string{"tenantId", "isDeleted", "dataset_id", "creator", "updater"}
needUpdate := false needUpdate := false
// 检查是否缺少必要的 filterable attributes // 检查是否缺少必要的 filterable attributes
@@ -226,6 +226,8 @@ func (m *meilisearchDB) buildSearchRequest(ctx context.Context, searchParams *Se
req.AttributesToRetrieve = searchParams.AttributesToRetrieve req.AttributesToRetrieve = searchParams.AttributesToRetrieve
} }
req.ShowRankingScore = searchParams.ShowRankingScore
return req, nil return req, nil
} }

View File

@@ -15,6 +15,7 @@ type SearchParams struct {
AttributesToRetrieve []string // 返回字段 AttributesToRetrieve []string // 返回字段
Facets []string // 聚合字段 Facets []string // 聚合字段
HitsPerPage int // 每页命中数 HitsPerPage int // 每页命中数
ShowRankingScore bool // 是否显示排名分数
} }
// SearchResult 搜索结果 // SearchResult 搜索结果

2
go.mod
View File

@@ -5,6 +5,7 @@ go 1.26.0
require ( require (
github.com/alibaba/sentinel-golang v1.0.4 github.com/alibaba/sentinel-golang v1.0.4
github.com/bwmarrin/snowflake v0.3.0 github.com/bwmarrin/snowflake v0.3.0
github.com/go-ego/gse v1.0.2
github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5
github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5
github.com/gogf/gf/v2 v2.9.5 github.com/gogf/gf/v2 v2.9.5
@@ -125,6 +126,7 @@ require (
github.com/tklauser/go-sysconf v0.3.6 // indirect github.com/tklauser/go-sysconf v0.3.6 // indirect
github.com/tklauser/numcpus v0.2.2 // indirect github.com/tklauser/numcpus v0.2.2 // indirect
github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/fastrand v1.1.0 // indirect
github.com/vcaesar/cedar v0.30.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect

6
go.sum
View File

@@ -125,6 +125,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-ego/gse v1.0.2 h1:+27lYFPhQEhA9igtdOsJPRKYL/k3TwYsxBF5jr6KFv4=
github.com/go-ego/gse v1.0.2/go.mod h1:Fy35G+q7VV7Et1zIKO8o/sW1kkugV3znXap/lF/11zc=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -591,6 +593,10 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/vcaesar/cedar v0.30.0 h1:9fSDpM7FTjjUdPiBUUa0MWYMRGSEcqgFXvppZcZ4d7Y=
github.com/vcaesar/cedar v0.30.0/go.mod h1:lyuGvALuZZDPNXwpzv/9LyxW+8Y6faN7zauFezNsnik=
github.com/vcaesar/tt v0.20.1 h1:D/jUeeVCNbq3ad8M7hhtB3J9x5RZ6I1n1eZ0BJp7M+4=
github.com/vcaesar/tt v0.20.1/go.mod h1:cH2+AwGAJm19Wa6xvEa+0r+sXDJBT0QgNQey6mwqLeU=
github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=

View File

@@ -62,7 +62,7 @@ func IPLimiter(r *ghttp.Request) {
// UserLimiter 用户维度限流中间件(防止单用户滥用) // UserLimiter 用户维度限流中间件(防止单用户滥用)
func UserLimiter(r *ghttp.Request) { func UserLimiter(r *ghttp.Request) {
if r.RequestURI == "/swagger" || r.RequestURI == "/admin-go/api/v1/pub/captcha/get" || r.RequestURI == "/admin-go/api/v1/system/login" { if strings.Contains(r.RequestURI, "/swagger") || strings.Contains(r.RequestURI, "/pub/captcha/get") || strings.Contains(r.RequestURI, "/login") || strings.Contains(r.RequestURI, "/web/socket/") {
r.Middleware.Next() r.Middleware.Next()
return return
} }

122
utils/gse.go Normal file
View File

@@ -0,0 +1,122 @@
package utils
import (
"context"
"sort"
"sync"
"github.com/go-ego/gse"
"github.com/go-ego/gse/hmm/extracker"
"github.com/go-ego/gse/hmm/segment"
"github.com/gogf/gf/v2/os/glog"
)
// 全局工具实例(不再自动初始化)
var (
GseTool *gseTool
once sync.Once // 保证只初始化一次,线程安全
)
func InitGseTool(ctx context.Context) error {
var err error
once.Do(func() {
// 只执行一次初始化
GseTool, err = newGseTool()
})
if err != nil {
glog.Error(ctx, "gse 分词工具初始化失败:", err)
}
return err
}
// gseTool 关键词提取工具gse v1.0.2 标准)
type gseTool struct {
seg gse.Segmenter
tfidf *extracker.TagExtracter
tr *extracker.TextRanker
}
// newGseTool 初始化工具(内置词典 + 停用词)
func newGseTool() (tool *gseTool, err error) {
// 1. 初始化分词器
var seg gse.Segmenter
// 内置词典(无外部文件)
err = seg.LoadDictEmbed()
if err != nil {
return
}
// 内置停用词v1.0.2 标准)
err = seg.LoadStopEmbed()
if err != nil {
return
}
// 2. 初始化 TF-IDF 提取器
tfidf := &extracker.TagExtracter{}
tfidf.WithGse(seg)
err = tfidf.LoadIdf()
if err != nil {
return
}
// 3. 初始化 TextRank 提取器
tr := &extracker.TextRanker{}
tr.WithGse(seg)
tool = &gseTool{
seg: seg,
tfidf: tfidf,
tr: tr,
}
return
}
// Cut 分词(关键词提取唯一正确模式:精确模式 + HMM
func (k *gseTool) Cut(text string) []string {
return k.seg.Cut(text, true)
}
// Keyword 最终输出:关键词 + 权重
type Keyword struct {
Word string `json:"word"`
Score float64 `json:"score"`
}
func (k *gseTool) Extract(text string, topN int) []Keyword {
// 1. 提取 TF-IDF
tfTags := k.extractTFIDF(text, topN)
// 2. 提取 TextRank
trTags := k.extractTextRank(text, topN)
// 3. 合并成最终关键词(业务最常用)
scoreMap := make(map[string]float64)
for _, tag := range tfTags {
scoreMap[tag.Text] = tag.Weight
}
for _, tag := range trTags {
scoreMap[tag.Text] = tag.Weight
}
// 转成切片并排序(高分在前)
res := make([]Keyword, 0, len(scoreMap))
for word, score := range scoreMap {
res = append(res, Keyword{Word: word, Score: score})
}
sort.Slice(res, func(i, j int) bool {
return res[i].Score > res[j].Score
})
return res
}
// ExtractTFIDF TF-IDF 关键词带权重90% 业务:文章标签、搜索、关键词
func (k *gseTool) extractTFIDF(text string, topN int) segment.Segments {
return k.tfidf.ExtractTags(text, topN)
}
// ExtractTextRank TextRank 关键词(带权重)长文本、摘要、语义理解
func (k *gseTool) extractTextRank(text string, topN int) segment.Segments {
return k.tr.TextRank(text, topN)
}