fix: optimize operation log resolve flow and i18n replacements (#12123)

This commit is contained in:
ssongliu
2026-03-09 16:02:12 +08:00
committed by GitHub
parent aa8a9e8b1e
commit 740f3549ad
8 changed files with 4074 additions and 80 deletions

View File

@@ -0,0 +1,6 @@
package docs
import _ "embed"
//go:embed x-log.json
var XLogJson []byte

File diff suppressed because it is too large Load Diff

View File

@@ -21,6 +21,7 @@ func Routers() *gin.Engine {
if !global.IsMaster {
PrivateGroup.Use(middleware.Certificate())
}
PrivateGroup.Use(middleware.OperationResolveMeta())
for _, router := range rou.RouterGroupApp {
router.InitRouter(PrivateGroup)
}

View File

@@ -0,0 +1,206 @@
package middleware
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"mime"
"mime/multipart"
"net/http"
"path"
"strings"
"sync"
"time"
"github.com/1Panel-dev/1Panel/agent/cmd/server/docs"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/re"
"github.com/gin-gonic/gin"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
const (
headerNeedOperationResolve = "X-Need-Op-Resolve"
headerOperationResolved = "X-Op-Resolved"
)
var (
logMetaOnce sync.Once
logMetaData map[string]operationMeta
logMetaLoadErr error
)
type operationMeta struct {
BodyKeys []string `json:"bodyKeys"`
BeforeFunctions []functionInfo `json:"beforeFunctions"`
}
type functionInfo struct {
InputColumn string `json:"input_column"`
InputValue string `json:"input_value"`
IsList bool `json:"isList"`
DB string `json:"db"`
OutputColumn string `json:"output_column"`
OutputValue string `json:"output_value"`
}
func OperationResolveMeta() gin.HandlerFunc {
return func(c *gin.Context) {
if c.GetHeader(headerNeedOperationResolve) != "1" {
c.Next()
return
}
metaMap, err := loadOperationMeta()
if err != nil {
c.Next()
return
}
reqPath := strings.TrimPrefix(c.Request.URL.Path, "/api/v2")
meta, ok := metaMap[reqPath]
if !ok || len(meta.BeforeFunctions) == 0 {
c.Next()
return
}
values := make(map[string]interface{})
if len(meta.BodyKeys) > 0 {
body, err := io.ReadAll(c.Request.Body)
if err == nil {
c.Request.Body = io.NopCloser(bytes.NewBuffer(body))
bodyMap := make(map[string]interface{})
if strings.Contains(c.Request.Header.Get("Content-Type"), "multipart/form-data") {
bodyMap, _ = parseMultipart(body, c.Request.Header.Get("Content-Type"))
} else {
_ = json.Unmarshal(body, &bodyMap)
}
for _, key := range meta.BodyKeys {
if value, ok := bodyMap[key]; ok {
values[key] = value
}
}
}
}
resolved, err := resolveOperationValues(reqPath, values, meta.BeforeFunctions)
if err == nil && len(resolved) > 0 {
if data, err := json.Marshal(resolved); err == nil {
c.Header(headerOperationResolved, base64.RawURLEncoding.EncodeToString(data))
}
}
c.Next()
}
}
func loadOperationMeta() (map[string]operationMeta, error) {
logMetaOnce.Do(func() {
logMetaData = make(map[string]operationMeta)
logMetaLoadErr = json.Unmarshal(docs.XLogJson, &logMetaData)
})
return logMetaData, logMetaLoadErr
}
func parseMultipart(formData []byte, contentType string) (map[string]interface{}, error) {
d, params, err := mime.ParseMediaType(contentType)
if err != nil || d != "multipart/form-data" {
return nil, http.ErrNotMultipart
}
boundary, ok := params["boundary"]
if !ok {
return nil, http.ErrMissingBoundary
}
reader := multipart.NewReader(bytes.NewReader(formData), boundary)
ret := make(map[string]interface{})
f, err := reader.ReadForm(32 << 20)
if err != nil {
return nil, err
}
for k, v := range f.Value {
if len(v) > 0 {
ret[k] = v[0]
}
}
for k, v := range f.File {
if len(v) > 0 {
ret[k] = v[0].Filename
}
}
return ret, nil
}
func resolveOperationValues(pathItem string, values map[string]interface{}, beforeFunctions []functionInfo) (map[string]string, error) {
dbItem, err := newResolveDB(pathItem)
if err != nil {
return nil, err
}
defer closeResolveDB(dbItem)
resolved := make(map[string]string)
for _, funcs := range beforeFunctions {
if !isSafeIdentifier(funcs.DB) || !isSafeIdentifier(funcs.InputColumn) || !isSafeIdentifier(funcs.OutputColumn) {
continue
}
for key, value := range values {
if funcs.InputValue != key {
continue
}
var names []string
if funcs.IsList {
sql := fmt.Sprintf("SELECT %s FROM %s where %s in (?);", funcs.OutputColumn, funcs.DB, funcs.InputColumn)
_ = dbItem.Raw(sql, value).Scan(&names)
} else {
sql := fmt.Sprintf("SELECT %s FROM %s where %s = ?;", funcs.OutputColumn, funcs.DB, funcs.InputColumn)
_ = dbItem.Raw(sql, value).Scan(&names)
}
outputValue := strings.Join(names, ",")
resolved[funcs.OutputValue] = outputValue
values[funcs.OutputValue] = outputValue
break
}
}
return resolved, nil
}
func newResolveDB(pathItem string) (*gorm.DB, error) {
dbFile := ""
switch {
case strings.HasPrefix(pathItem, "/core"):
dbFile = path.Join(global.CONF.Base.InstallDir, "1panel/db/core.db")
case strings.HasPrefix(pathItem, "/xpack"):
dbFile = path.Join(global.CONF.Base.InstallDir, "1panel/db/xpack.db")
default:
dbFile = path.Join(global.CONF.Base.InstallDir, "1panel/db/agent.db")
}
db, err := gorm.Open(sqlite.Open(dbFile), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
if err != nil {
return nil, err
}
sqlDB, err := db.DB()
if err != nil {
return nil, err
}
sqlDB.SetMaxOpenConns(4)
sqlDB.SetMaxIdleConns(1)
sqlDB.SetConnMaxIdleTime(15 * time.Minute)
sqlDB.SetConnMaxLifetime(time.Hour)
return db, nil
}
func closeResolveDB(db *gorm.DB) {
sqlDB, err := db.DB()
if err != nil {
return
}
_ = sqlDB.Close()
}
func isSafeIdentifier(val string) bool {
return re.GetRegex(re.SQLIdentifierPattern).MatchString(val)
}

View File

@@ -29,6 +29,7 @@ const (
AnsiEscapePattern = "\x1b\\[[0-9;?]*[A-Za-z]|\x1b=|\x1b>"
RecycleBinFilePattern = `_1p_file_1p_(.+)_p_(\d+)_(\d+)`
OrderByValidationPattern = `^[a-zA-Z_][a-zA-Z0-9_]*$`
SQLIdentifierPattern = `^[A-Za-z_][A-Za-z0-9_]*$`
NginxHostPattern = `^[a-zA-Z0-9.-]+(:[0-9]+)?$`
NginxPathPattern = `^/[a-zA-Z0-9._/\-]*$`
)
@@ -60,6 +61,7 @@ func Init() {
AnsiEscapePattern,
RecycleBinFilePattern,
OrderByValidationPattern,
SQLIdentifierPattern,
NginxHostPattern,
NginxPathPattern,
}

View File

@@ -62,7 +62,10 @@ func TestGenerateXlog(t *testing.T) {
panic(fmt.Sprintf("json marshal for new file failed, err: %v", err))
}
if err := os.WriteFile("x-log.json", newJson, 0640); err != nil {
panic(fmt.Sprintf("write new swagger.json failed, err: %v", err))
panic(fmt.Sprintf("write core x-log.json failed, err: %v", err))
}
if err := os.WriteFile(workDir+"/agent/cmd/server/docs/x-log.json", newJson, 0640); err != nil {
panic(fmt.Sprintf("write agent x-log.json failed, err: %v", err))
}
}

View File

@@ -3,6 +3,7 @@ package middleware
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -25,8 +26,15 @@ import (
"gorm.io/gorm"
)
const (
headerNeedOperationResolve = "X-Need-Op-Resolve"
headerOperationResolved = "X-Op-Resolved"
)
func OperationLog() gin.HandlerFunc {
return func(c *gin.Context) {
c.Request.Header.Del(headerNeedOperationResolve)
if strings.Contains(c.Request.URL.Path, "search") || c.Request.Method == http.MethodGet {
c.Next()
return
@@ -78,61 +86,43 @@ func OperationLog() gin.HandlerFunc {
}
}
}
if len(operationDic.BeforeFunctions) != 0 {
dbItem, err := newDB(record.Path)
if err != nil {
c.Next()
return
}
for _, funcs := range operationDic.BeforeFunctions {
for key, value := range formatMap {
if funcs.InputValue == key {
var names []string
if funcs.IsList {
sql := fmt.Sprintf("SELECT %s FROM %s where %s in (?);", funcs.OutputColumn, funcs.DB, funcs.InputColumn)
_ = dbItem.Raw(sql, value).Scan(&names)
} else {
_ = dbItem.Raw(fmt.Sprintf("select %s from %s where %s = ?;", funcs.OutputColumn, funcs.DB, funcs.InputColumn), value).Scan(&names)
}
formatMap[funcs.OutputValue] = strings.Join(names, ",")
break
}
}
}
closeDB(dbItem)
needAgentResolve := len(operationDic.BeforeFunctions) != 0 &&
(len(currentNode) == 0 || currentNode == "local") &&
!strings.HasPrefix(record.Path, "/core")
allowCoreFallback := strings.HasPrefix(record.Path, "/core/xpack") || !willProxy(c.Request.URL.Path, currentNode)
if needAgentResolve {
c.Request.Header.Set(headerNeedOperationResolve, "1")
defer func() {
c.Request.Header.Del(headerNeedOperationResolve)
}()
}
for key, value := range formatMap {
if strings.Contains(operationDic.FormatEN, "["+key+"]") {
t := reflect.TypeOf(value)
if t.Kind() != reflect.Array && t.Kind() != reflect.Slice {
operationDic.FormatZH = strings.ReplaceAll(operationDic.FormatZH, "["+key+"]", fmt.Sprintf("[%v]", value))
operationDic.FormatEN = strings.ReplaceAll(operationDic.FormatEN, "["+key+"]", fmt.Sprintf("[%v]", value))
} else {
val := reflect.ValueOf(value)
length := val.Len()
var elements []string
for i := 0; i < length; i++ {
element := val.Index(i).Interface().(string)
elements = append(elements, element)
}
operationDic.FormatZH = strings.ReplaceAll(operationDic.FormatZH, "["+key+"]", fmt.Sprintf("[%v]", strings.Join(elements, ",")))
operationDic.FormatEN = strings.ReplaceAll(operationDic.FormatEN, "["+key+"]", fmt.Sprintf("[%v]", strings.Join(elements, ",")))
}
}
}
record.DetailEN = strings.ReplaceAll(operationDic.FormatEN, "[]", "")
record.DetailZH = strings.ReplaceAll(operationDic.FormatZH, "[]", "")
writer := responseBodyWriter{
ResponseWriter: c.Writer,
body: &bytes.Buffer{},
}
c.Writer = writer
c.Writer = &writer
now := time.Now()
c.Next()
if len(operationDic.BeforeFunctions) != 0 {
if needAgentResolve {
mergeResolvedData(writer.resolvedHeader, formatMap)
}
if allowCoreFallback && !hasAllResolvedData(formatMap, operationDic.BeforeFunctions) {
dbItem, err := newDB(record.Path)
if err == nil {
resolveByDB(dbItem, formatMap, operationDic.BeforeFunctions)
closeDB(dbItem)
}
}
}
fillOperationDetail(&operationDic, formatMap)
record.DetailEN = strings.ReplaceAll(operationDic.FormatEN, "[]", "")
record.DetailZH = strings.ReplaceAll(operationDic.FormatZH, "[]", "")
datas := writer.body.Bytes()
logRepo := repo.NewILogRepo()
if c.Request.Header.Get("Content-Encoding") == "gzip" {
@@ -170,6 +160,30 @@ func OperationLog() gin.HandlerFunc {
}
}
func fillOperationDetail(operationDic *operationJson, formatMap map[string]interface{}) {
for key, value := range formatMap {
if !strings.Contains(operationDic.FormatEN, "["+key+"]") {
continue
}
t := reflect.TypeOf(value)
if t == nil || (t.Kind() != reflect.Array && t.Kind() != reflect.Slice) {
operationDic.FormatZH = strings.ReplaceAll(operationDic.FormatZH, "["+key+"]", fmt.Sprintf("[%v]", value))
operationDic.FormatEN = strings.ReplaceAll(operationDic.FormatEN, "["+key+"]", fmt.Sprintf("[%v]", value))
continue
}
val := reflect.ValueOf(value)
length := val.Len()
elements := make([]string, 0, length)
for i := 0; i < length; i++ {
elements = append(elements, fmt.Sprintf("%v", val.Index(i).Interface()))
}
replaced := fmt.Sprintf("[%v]", strings.Join(elements, ","))
operationDic.FormatZH = strings.ReplaceAll(operationDic.FormatZH, "["+key+"]", replaced)
operationDic.FormatEN = strings.ReplaceAll(operationDic.FormatEN, "["+key+"]", replaced)
}
}
type operationJson struct {
API string `json:"api"`
Method string `json:"method"`
@@ -195,10 +209,29 @@ type response struct {
type responseBodyWriter struct {
gin.ResponseWriter
body *bytes.Buffer
body *bytes.Buffer
resolvedHeader string
}
func (r responseBodyWriter) Write(b []byte) (int, error) {
func (r *responseBodyWriter) sanitizeResolvedHeader() {
if len(r.resolvedHeader) == 0 {
r.resolvedHeader = r.ResponseWriter.Header().Get(headerOperationResolved)
}
r.ResponseWriter.Header().Del(headerOperationResolved)
}
func (r *responseBodyWriter) WriteHeader(code int) {
r.sanitizeResolvedHeader()
r.ResponseWriter.WriteHeader(code)
}
func (r *responseBodyWriter) WriteHeaderNow() {
r.sanitizeResolvedHeader()
r.ResponseWriter.WriteHeaderNow()
}
func (r *responseBodyWriter) Write(b []byte) (int, error) {
r.sanitizeResolvedHeader()
r.body.Write(b)
return r.ResponseWriter.Write(b)
}
@@ -218,10 +251,10 @@ func loadLogInfo(path string) string {
func newDB(pathItem string) (*gorm.DB, error) {
dbFile := ""
switch {
case strings.HasPrefix(pathItem, "/core/xpack") || strings.HasPrefix(pathItem, "/xpack"):
dbFile = path.Join(global.CONF.Base.InstallDir, "1panel/db/xpack.db")
case strings.HasPrefix(pathItem, "/core"):
dbFile = path.Join(global.CONF.Base.InstallDir, "1panel/db/core.db")
case strings.HasPrefix(pathItem, "/xpack"):
dbFile = path.Join(global.CONF.Base.InstallDir, "1panel/db/xpack.db")
default:
dbFile = path.Join(global.CONF.Base.InstallDir, "1panel/db/agent.db")
}
@@ -248,6 +281,26 @@ func closeDB(db *gorm.DB) {
_ = sqlDB.Close()
}
func resolveByDB(dbItem *gorm.DB, values map[string]interface{}, beforeFunctions []functionInfo) {
for _, funcs := range beforeFunctions {
for key, value := range values {
if funcs.InputValue != key {
continue
}
var names []string
if funcs.IsList {
sql := fmt.Sprintf("SELECT %s FROM %s where %s in (?);", funcs.OutputColumn, funcs.DB, funcs.InputColumn)
_ = dbItem.Raw(sql, value).Scan(&names)
} else {
sql := fmt.Sprintf("select %s from %s where %s = ?;", funcs.OutputColumn, funcs.DB, funcs.InputColumn)
_ = dbItem.Raw(sql, value).Scan(&names)
}
values[funcs.OutputValue] = strings.Join(names, ",")
break
}
}
}
func replaceStr(val string, rep ...string) string {
for _, item := range rep {
val = strings.ReplaceAll(val, item, "")
@@ -284,3 +337,43 @@ func parseMultipart(formData []byte, contentType string) (map[string]interface{}
}
return ret, nil
}
func mergeResolvedData(headerVal string, values map[string]interface{}) {
if len(headerVal) == 0 {
return
}
data, err := base64.RawURLEncoding.DecodeString(headerVal)
if err != nil {
return
}
resolved := make(map[string]string)
if err := json.Unmarshal(data, &resolved); err != nil {
return
}
for key, value := range resolved {
values[key] = value
}
}
func hasAllResolvedData(values map[string]interface{}, beforeFunctions []functionInfo) bool {
for _, item := range beforeFunctions {
if _, ok := values[item.OutputValue]; ok {
continue
}
return false
}
return true
}
func willProxy(reqPath, currentNode string) bool {
if strings.HasPrefix(reqPath, "/1panel/swagger") || !strings.HasPrefix(reqPath, "/api/v2") {
return false
}
if strings.HasPrefix(reqPath, "/api/v2/core") && !strings.HasPrefix(reqPath, "/api/v2/core/xpack") {
return false
}
if !strings.HasPrefix(reqPath, "/api/v2/core") && (currentNode == "local" || len(currentNode) == 0) {
return true
}
return true
}

View File

@@ -156,12 +156,13 @@ const onClean = async () => {
};
const loadDetail = (log: string) => {
for (const [key, value] of Object.entries(replacements)) {
if (log.indexOf(key) !== -1) {
log = log.replace(key, '[' + i18n.global.t(value) + ']');
return log.replace(/\[([^\]]+)\]/g, (matched, token: string) => {
const transKey = resolveReplacementKey(token);
if (!transKey) {
return matched;
}
}
return log;
return '[' + i18n.global.t(transKey) + ']';
});
};
const loadNodes = async () => {
@@ -178,31 +179,103 @@ const loadNodes = async () => {
});
};
const replacements = {
'[enable]': 'commons.button.enable',
'[Enable]': 'commons.button.enable',
'[disable]': 'commons.button.disable',
'[Disable]': 'commons.button.disable',
'[disableBanPing]': 'firewall.disableBanPing',
'[enableBanPing]': 'firewall.enableBanPing',
'[light]': 'setting.light',
'[dark]': 'setting.dark',
'[delete]': 'commons.button.delete',
'[get]': 'commons.button.get',
'[operate]': 'commons.table.operate',
'[UserName]': 'commons.login.username',
'[PanelName]': 'setting.title',
'[Language]': 'setting.language',
'[Theme]': 'setting.theme',
'[MenuTabs]': 'setting.menuTabs',
'[SessionTimeout]': 'setting.sessionTimeout',
'[SecurityEntrance]': 'setting.entrance',
'[ExpirationDays]': 'setting.expirationTime',
'[ComplexityVerification]': 'setting.complexity',
'[MFAStatus]': 'setting.mfa',
'[MonitorStatus]': 'setting.enableMonitor',
'[MonitorStoreDays]': 'setting.monitor',
'[ApiInterfaceStatus]': 'setting.apiInterface',
const normalizedReplacements: Record<string, string> = {
enable: 'commons.button.enable',
disable: 'commons.button.disable',
start: 'commons.button.start',
stop: 'commons.button.stop',
restart: 'commons.button.restart',
reload: 'commons.operate.reload',
sync: 'commons.button.sync',
update: 'commons.button.update',
open: 'commons.button.open',
close: 'commons.button.close',
up: 'commons.button.up',
down: 'commons.button.down',
login: 'commons.button.login',
delete: 'commons.button.delete',
create: 'commons.button.create',
add: 'commons.button.add',
edit: 'commons.button.edit',
save: 'commons.button.save',
clean: 'commons.button.clean',
clear: 'commons.button.clean',
get: 'commons.button.get',
install: 'commons.button.install',
uninstall: 'commons.button.uninstall',
backup: 'commons.button.backup',
recover: 'commons.button.recover',
upload: 'commons.button.upload',
download: 'commons.button.download',
bind: 'commons.button.bind',
unbind: 'commons.button.unbind',
verify: 'commons.button.verify',
remove: 'commons.msg.remove',
kill: 'container.kill',
pause: 'container.pause',
unpause: 'container.unpause',
allow: 'firewall.allow',
deny: 'firewall.deny',
accept: 'firewall.accept',
drop: 'firewall.drop',
reject: 'firewall.stop',
running: 'commons.status.running',
stopped: 'commons.status.stopped',
success: 'commons.status.success',
failed: 'commons.status.failed',
created: 'commons.status.created',
restarting: 'commons.status.restarting',
paused: 'commons.status.paused',
exited: 'commons.status.exited',
dead: 'commons.status.dead',
light: 'setting.light',
dark: 'setting.dark',
darkgold: 'setting.darkGold',
auto: 'setting.auto',
cn: 'setting.cn',
intl: 'setting.intl',
status: 'commons.table.status',
all: 'commons.table.all',
operate: 'commons.table.operate',
true: 'commons.true',
false: 'commons.false',
};
const exactReplacements: Record<string, string> = {
disableBanPing: 'firewall.disableBanPing',
enableBanPing: 'firewall.enableBanPing',
UserName: 'commons.login.username',
PanelName: 'setting.title',
Language: 'setting.language',
Theme: 'setting.theme',
MenuTabs: 'setting.menuTabs',
SessionTimeout: 'setting.sessionTimeout',
SecurityEntrance: 'setting.entrance',
ExpirationDays: 'setting.expirationTime',
ComplexityVerification: 'setting.complexity',
MFAStatus: 'setting.mfa',
MonitorStatus: 'setting.enableMonitor',
MonitorStoreDays: 'setting.monitor',
ApiInterfaceStatus: 'setting.apiInterface',
ComponentSize: 'setting.componentSize',
Region: 'setting.region',
SystemIP: 'setting.systemIP',
ProxyType: 'setting.proxyType',
ProxyUrl: 'setting.proxyUrl',
ProxyPort: 'setting.proxyPort',
ProxyPasswdKeep: 'setting.proxyPasswdKeep',
ProxyDocker: 'setting.proxyDocker',
SyncToNode: 'setting.syncToNode',
IPWhiteList: 'setting.ipWhiteList',
ApiKeyValidityTime: 'setting.apiKeyValidityTime',
DeveloperMode: 'setting.developerMode',
};
const resolveReplacementKey = (token: string): string | undefined => {
if (exactReplacements[token]) {
return exactReplacements[token];
}
return normalizedReplacements[token.toLowerCase()];
};
const onSubmitClean = async () => {