1Panel/agent/app/service/backup_mongodb.go

package service

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/url"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/1Panel-dev/1Panel/agent/app/dto"
	"github.com/1Panel-dev/1Panel/agent/app/model"
	"github.com/1Panel-dev/1Panel/agent/app/repo"
	"github.com/1Panel-dev/1Panel/agent/app/task"
	"github.com/1Panel-dev/1Panel/agent/buserr"
	"github.com/1Panel-dev/1Panel/agent/constant"
	"github.com/1Panel-dev/1Panel/agent/global"
	"github.com/1Panel-dev/1Panel/agent/i18n"
	"github.com/1Panel-dev/1Panel/agent/utils/cmd"
	"github.com/1Panel-dev/1Panel/agent/utils/common"
	"github.com/1Panel-dev/1Panel/agent/utils/files"
	dockerImage "github.com/docker/docker/api/types/image"
	dockerClient "github.com/docker/docker/client"
)

const opensslSaltedHeader = "Salted__"

func (u *BackupService) MongodbBackup(req dto.CommonBackup) error {
	timeNow := time.Now().Format(constant.DateTimeSlimLayout)
	itemDir := fmt.Sprintf("database/%s/%s/%s", req.Type, req.Name, req.DetailName)
	targetDir := path.Join(global.Dir.LocalBackupDir, itemDir)
	fileName := fmt.Sprintf("%s_%s.gz", req.DetailName, timeNow+common.RandStrAndNum(5))

	record := &model.BackupRecord{
		Type:              req.Type,
		Name:              req.Name,
		DetailName:        req.DetailName,
		SourceAccountIDs:  "1",
		DownloadAccountID: 1,
		FileDir:           itemDir,
		FileName:          fileName,
		TaskID:            req.TaskID,
		Status:            constant.StatusWaiting,
		Description:       req.Description,
	}
	if err := backupRepo.CreateRecord(record); err != nil {
		global.LOG.Errorf("save backup record failed, err: %v", err)
		return err
	}

	if err := handleMongodbBackup(req, nil, record.ID, targetDir, fileName); err != nil {
		markBackupFailed(record.ID, err)
		return err
	}
	return nil
}

func (u *BackupService) MongodbRecover(req dto.CommonRecover) error {
	return handleMongodbRecover(req, nil, false)
}

func (u *BackupService) MongodbRecoverByUpload(req dto.CommonRecover) error {
	return handleMongodbRecover(req, nil, false)
}

func handleMongodbBackup(req dto.CommonBackup, parentTask *task.Task, recordID uint, targetDir, fileName string) error {
	dbItem, err := mongodbRepo.Get(repo.WithByName(req.DetailName), mongodbRepo.WithByMongodbName(req.Name))
	if err != nil {
		return err
	}
	itemName := fmt.Sprintf("%s[%s] - %s", req.Name, req.Type, req.DetailName)
	backupTask := parentTask
	if backupTask == nil {
		backupTask, err = task.NewTaskWithOps(itemName, task.TaskBackup, task.TaskScopeBackup, req.TaskID, dbItem.ID)
		if err != nil {
			return err
		}
	}

	itemHandler := func(t *task.Task) error {
		return doMongodbBackup(req.Name, req.Type, req.DetailName, targetDir, fileName, req.Secret, t)
	}
	if parentTask != nil {
		return itemHandler(parentTask)
	}

	backupTask.AddSubTaskWithOps(
		task.GetTaskName(itemName, task.TaskBackup, task.TaskScopeBackup),
		func(t *task.Task) error { return itemHandler(t) },
		nil,
		0,
		3*time.Hour,
	)
	go func() {
		if err := backupTask.Execute(); err != nil {
			markBackupFailed(recordID, err)
			return
		}
		backupRepo.UpdateRecordByMap(recordID, map[string]interface{}{"status": constant.StatusSuccess})
	}()
	return nil
}

func handleMongodbRecover(req dto.CommonRecover, parentTask *task.Task, isRollback bool) error {
	dbItem, err := mongodbRepo.Get(repo.WithByName(req.DetailName), mongodbRepo.WithByMongodbName(req.Name))
	if err != nil {
		return err
	}
	itemName := fmt.Sprintf("%s[%s] - %s", req.Name, req.Type, req.DetailName)
	recoverTask := parentTask
	if recoverTask == nil {
		recoverTask, err = task.NewTaskWithOps(itemName, task.TaskRecover, task.TaskScopeBackup, req.TaskID, dbItem.ID)
		if err != nil {
			return err
		}
	}

	recoverDatabase := func(t *task.Task) error {
		fileOp := files.NewFileOp()
		if !fileOp.Stat(req.File) {
			return buserr.WithName("ErrFileNotFound", req.File)
		}

		restoreFile, cleanup, err := prepareMongodbBackupFileForRestore(req.File, req.Secret, t)
		if err != nil {
			return err
		}
		defer cleanup()

		isOk := false
		if !isRollback {
			rollbackFile := path.Join(
				global.Dir.TmpDir,
				fmt.Sprintf("database/%s/%s_%s.gz", req.Type, req.DetailName, time.Now().Format(constant.DateTimeSlimLayout)),
			)
			if err := doMongodbBackup(req.Name, req.Type, req.DetailName, path.Dir(rollbackFile), path.Base(rollbackFile), "", t); err != nil {
				return fmt.Errorf("backup mongodb db %s for rollback before recover failed, err: %v", req.DetailName, err)
			}
			defer func() {
				if !isOk {
					global.LOG.Info("recover failed, start to rollback now")
					if err := doMongodbRestore(req.Name, req.Type, req.DetailName, rollbackFile, t); err != nil {
						global.LOG.Errorf("rollback mongodb db %s from %s failed, err: %v", req.DetailName, rollbackFile, err)
					} else {
						global.LOG.Infof("rollback mongodb db %s from %s successful", req.DetailName, rollbackFile)
					}
				}
				_ = os.RemoveAll(rollbackFile)
			}()
		}

		if req.DropAllCollections {
			if err := clearMongodbDatabase(req.Name, req.Type, req.DetailName, t); err != nil {
				return err
			}
		}

		if err := doMongodbRestore(req.Name, req.Type, req.DetailName, restoreFile, t); err != nil {
			global.LOG.Errorf("recover mongodb db %s from %s failed, err: %v", req.DetailName, restoreFile, err)
			return err
		}
		isOk = true
		return nil
	}
	if parentTask != nil {
		return recoverDatabase(parentTask)
	}

	var timeout time.Duration
	switch req.Timeout {
	case -1:
		timeout = 0
	case 0:
		timeout = 3 * time.Hour
	default:
		timeout = time.Duration(req.Timeout) * time.Second
	}
	recoverTask.AddSubTaskWithOps(i18n.GetMsgByKey("TaskRecover"), recoverDatabase, nil, 0, timeout)
	go func() {
		_ = recoverTask.Execute()
	}()
	return nil
}

func doMongodbBackup(database, dbType, dbName, targetDir, fileName, secret string, taskItem *task.Task) error {
	dbItem, err := mongodbRepo.Get(repo.WithByName(dbName), mongodbRepo.WithByMongodbName(database))
	if err == nil && dbItem.From == constant.AppResourceRemote {
		if err := doRemoteMongodbBackup(database, dbName, targetDir, fileName, taskItem); err != nil {
			return err
		}
		if len(secret) != 0 {
			return files.OpensslEncrypt(path.Join(targetDir, fileName), secret)
		}
		return nil
	}

	appInfo, err := appInstallRepo.LoadBaseInfo(dbType, database)
	if err != nil {
		return err
	}
	if appInfo.ContainerName == "" {
		return fmt.Errorf("mongodb container not found for database %s", database)
	}
	if err := os.MkdirAll(targetDir, constant.DirPerm); err != nil {
		return err
	}

	targetFile := path.Join(targetDir, fileName)
	containerFile := path.Join("/tmp", fileName)
	defer func() {
		_ = cmd.NewCommandMgr().Run("docker", "exec", appInfo.ContainerName, "rm", "-f", containerFile)
	}()

	uri := buildMongodbDumpURI(appInfo.UserName, appInfo.Password, dbName)
	cmdMgr := mongodbCmdMgr(taskItem)
	if err := cmdMgr.Run(
		"docker",
		"exec",
		appInfo.ContainerName,
		"mongodump",
		"--uri="+uri,
		"--archive="+containerFile,
		"--gzip",
	); err != nil {
		return err
	}
	if err := cmdMgr.Run("docker", "cp", fmt.Sprintf("%s:%s", appInfo.ContainerName, containerFile), targetFile); err != nil {
		return err
	}
	if len(secret) != 0 {
		return files.OpensslEncrypt(targetFile, secret)
	}
	return nil
}

func doMongodbRestore(database, dbType, dbName, sourceFile string, taskItem *task.Task) error {
	dbItem, err := mongodbRepo.Get(repo.WithByName(dbName), mongodbRepo.WithByMongodbName(database))
	if err == nil && dbItem.From == constant.AppResourceRemote {
		return doRemoteMongodbRestore(database, dbName, sourceFile, taskItem)
	}

	appInfo, err := appInstallRepo.LoadBaseInfo(dbType, database)
	if err != nil {
		return err
	}
	if appInfo.ContainerName == "" {
		return fmt.Errorf("mongodb container not found for database %s", database)
	}

	containerFile := path.Join("/tmp", fmt.Sprintf("1panel-mongodb-restore-%s.gz", common.RandStrAndNum(8)))
	defer func() {
		_ = cmd.NewCommandMgr().Run("docker", "exec", appInfo.ContainerName, "rm", "-f", containerFile)
	}()

	cmdMgr := mongodbCmdMgr(taskItem)
	if err := cmdMgr.Run("docker", "cp", sourceFile, fmt.Sprintf("%s:%s", appInfo.ContainerName, containerFile)); err != nil {
		return err
	}

	uri := buildMongodbRestoreURI(appInfo.UserName, appInfo.Password)
	if err := cmdMgr.Run(
		"docker",
		"exec",
		appInfo.ContainerName,
		"mongorestore",
		"--uri="+uri,
		"--nsInclude="+buildMongodbNamespace(sourceFile, dbName),
		"--nsFrom="+buildMongodbNamespace(sourceFile, dbName),
		"--nsTo="+dbName+".*",
		"--archive="+containerFile,
		"--gzip",
		"--drop",
	); err != nil {
		return err
	}
	return nil
}

func prepareMongodbBackupFileForRestore(filePath, secret string, taskItem *task.Task) (string, func(), error) {
	isEncrypted, err := isOpenSSLEncryptedMongodbBackup(filePath)
	if err != nil {
		return "", nil, err
	}
	if !isEncrypted {
		return filePath, func() {}, nil
	}

	if secret == "" {
		return "", nil, buserr.New("ErrBadDecrypt")
	}

	if err := files.OpensslDecrypt(filePath, secret); err != nil {
		return "", nil, err
	}

	restoreFile := path.Join(path.Dir(filePath), "tmp_"+path.Base(filePath))
	taskItem.LogWithStatus(i18n.GetMsgByKey("Decrypt"), nil)
	return restoreFile, func() { _ = os.Remove(restoreFile) }, nil
}

func isOpenSSLEncryptedMongodbBackup(filePath string) (bool, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return false, err
	}
	defer file.Close()

	header := make([]byte, len(opensslSaltedHeader))
	n, err := io.ReadFull(file, header)
	if err != nil {
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			return false, nil
		}
		return false, err
	}
	return n == len(opensslSaltedHeader) && string(header) == opensslSaltedHeader, nil
}

func clearMongodbDatabase(database, dbType, dbName string, taskItem *task.Task) error {
	dbItem, err := mongodbRepo.Get(repo.WithByName(dbName), mongodbRepo.WithByMongodbName(database))
	if err == nil && dbItem.From == constant.AppResourceRemote {
		return clearRemoteMongodbDatabase(database, dbName, taskItem)
	}

	appInfo, err := appInstallRepo.LoadBaseInfo(dbType, database)
	if err != nil {
		return err
	}
	if appInfo.ContainerName == "" {
		return fmt.Errorf("mongodb container not found for database %s", database)
	}

	logRemoteMongodbStep(taskItem, fmt.Sprintf("clear local mongodb database %s before restore", dbName))
	uri := buildMongodbRestoreURI(appInfo.UserName, appInfo.Password)
	return mongodbCmdMgr(taskItem).Run(
		"docker",
		"exec",
		appInfo.ContainerName,
		"mongosh",
		uri,
		"--quiet",
		"--eval",
		fmt.Sprintf(`db.getSiblingDB(%q).dropDatabase()`, dbName),
	)
}

func clearRemoteMongodbDatabase(database, dbName string, taskItem *task.Task) error {
	info, err := loadRemoteMongodbConnection(database)
	if err != nil {
		return err
	}
	client, ctx, cancel, err := newRemoteMongodbClient(info)
	if err != nil {
		return err
	}
	defer cancel()
	defer client.Disconnect(context.Background())

	logRemoteMongodbStep(taskItem, fmt.Sprintf("clear remote mongodb database %s before restore", dbName))
	if err := client.Database(dbName).Drop(ctx); err != nil {
		return fmt.Errorf("drop mongodb database %s failed, err: %v", dbName, err)
	}
	return nil
}

func buildMongodbDumpURI(username, password, dbName string) string {
	return (&url.URL{
		Scheme:   "mongodb",
		User:     url.UserPassword(username, password),
		Host:     "127.0.0.1:27017",
		Path:     "/" + dbName,
		RawQuery: "authSource=admin",
	}).String()
}

func buildMongodbRestoreURI(username, password string) string {
	return (&url.URL{
		Scheme:   "mongodb",
		User:     url.UserPassword(username, password),
		Host:     "127.0.0.1:27017",
		Path:     "/",
		RawQuery: "authSource=admin",
	}).String()
}

func mongodbCmdMgr(taskItem *task.Task) *cmd.CommandHelper {
	if taskItem == nil {
		return cmd.NewCommandMgr(cmd.WithTimeout(3 * time.Hour))
	}
	return cmd.NewCommandMgr(cmd.WithTimeout(3*time.Hour), cmd.WithTask(*taskItem))
}

func doRemoteMongodbBackup(database, dbName, targetDir, fileName string, taskItem *task.Task) error {
	info, err := loadRemoteMongodbConnection(database)
	if err != nil {
		return err
	}
	imageTag, err := ensureMongodbImage(database, taskItem)
	if err != nil {
		return err
	}
	logRemoteMongodbImage(taskItem, "backup", database, dbName, imageTag, info)
	logRemoteMongodbStep(taskItem, fmt.Sprintf("local image %s is ready, start backup", imageTag))
	if err := os.MkdirAll(targetDir, constant.DirPerm); err != nil {
		return err
	}

	targetFile, err := os.OpenFile(path.Join(targetDir, fileName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, constant.DirPerm)
	if err != nil {
		return fmt.Errorf("open file %s failed, err: %v", path.Join(targetDir, fileName), err)
	}
	defer func() { _ = targetFile.Close() }()

	backupCmd := exec.Command(
		"docker",
		"run",
		"--rm",
		"--net=host",
		"-i",
		imageTag,
		"mongodump",
		"--uri="+buildRemoteMongodbURI(info),
		"--db="+dbName,
		"--archive",
		"--gzip",
	)
	backupCmd.Stdout = targetFile
	var stderr bytes.Buffer
	backupCmd.Stderr = &stderr
	if err := backupCmd.Run(); err != nil {
		return fmt.Errorf("handle backup mongodb database failed, err: %s", strings.TrimSpace(stderr.String()))
	}
	return nil
}

func doRemoteMongodbRestore(database, dbName, sourceFile string, taskItem *task.Task) error {
	info, err := loadRemoteMongodbConnection(database)
	if err != nil {
		return err
	}
	imageTag, err := ensureMongodbImage(database, taskItem)
	if err != nil {
		return err
	}
	logRemoteMongodbImage(taskItem, "restore", database, dbName, imageTag, info)
	logRemoteMongodbStep(taskItem, fmt.Sprintf("local image %s is ready, start restore", imageTag))

	fi, err := os.Open(sourceFile)
	if err != nil {
		return err
	}
	defer func() { _ = fi.Close() }()

	restoreCmd := exec.Command(
		"docker",
		"run",
		"--rm",
		"--net=host",
		"-i",
		imageTag,
		"mongorestore",
		"--uri="+buildRemoteMongodbURI(info),
		"--nsInclude="+buildMongodbNamespace(sourceFile, dbName),
		"--nsFrom="+buildMongodbNamespace(sourceFile, dbName),
		"--nsTo="+dbName+".*",
		"--archive",
		"--gzip",
		"--drop",
	)
	restoreCmd.Stdin = fi
	var stderr bytes.Buffer
	restoreCmd.Stderr = &stderr
	if err := restoreCmd.Run(); err != nil {
		return fmt.Errorf("handle recover mongodb database failed, err: %s", strings.TrimSpace(stderr.String()))
	}
	return nil
}

func ensureMongodbImage(database string, taskItem *task.Task) (string, error) {
	imageTag, exists, err := loadMongodbImageTag(database)
	if err != nil {
		return "", err
	}
	logRemoteMongodbStep(taskItem, fmt.Sprintf("check local image %s", imageTag))
	if exists {
		logRemoteMongodbStep(taskItem, fmt.Sprintf("local image %s exists", imageTag))
		return imageTag, nil
	}
	logRemoteMongodbStep(taskItem, fmt.Sprintf("local image %s not found, start docker pull", imageTag))
	if err := mongodbCmdMgr(taskItem).Run("docker", "pull", imageTag); err != nil {
		return "", err
	}
	logRemoteMongodbStep(taskItem, fmt.Sprintf("docker pull %s finished", imageTag))
	return imageTag, nil
}

func loadMongodbImageTag(database string) (string, bool, error) {
	databaseInfo, err := databaseRepo.Get(repo.WithByName(database))
	if err != nil {
		return "", false, err
	}

	cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation())
	if err != nil {
		return "", false, err
	}
	defer cli.Close()

	images, err := cli.ImageList(context.Background(), dockerImage.ListOptions{})
	if err != nil {
		return "", false, err
	}
	imagePrefix := "mongo:" + loadMongodbImageMajor(databaseInfo.Version)
	for _, image := range images {
		for _, tag := range image.RepoTags {
			if strings.HasPrefix(tag, imagePrefix) {
				return tag, true, nil
			}
		}
	}
	return imagePrefix, false, nil
}

func buildMongodbNamespace(sourceFile, targetDB string) string {
	sourceDB := loadMongodbBackupDBName(sourceFile, targetDB)
	return sourceDB + ".*"
}

func loadMongodbBackupDBName(sourceFile, defaultDB string) string {
	baseName := path.Base(sourceFile)
	if strings.HasSuffix(baseName, ".gz") {
		baseName = strings.TrimSuffix(baseName, ".gz")
	}
	// Encrypted backups are restored from a decrypted temp file like tmp_<original>.
	// Strip the temp prefix before deriving the original database name.
	baseName = strings.TrimPrefix(baseName, "tmp_")
	patterns := []*regexp.Regexp{
		regexp.MustCompile(`^1panel_mongodb_(.+)_\d{14}[A-Za-z0-9]*$`),
		regexp.MustCompile(`^db_(.+)_\d{14}[A-Za-z0-9]*$`),
		regexp.MustCompile(`^(.+)_\d{14}[A-Za-z0-9]*$`),
	}
	for _, pattern := range patterns {
		matches := pattern.FindStringSubmatch(baseName)
		if len(matches) == 2 && len(matches[1]) != 0 {
			return matches[1]
		}
	}
	return defaultDB
}

func loadMongodbImageMajor(version string) string {
	switch {
	case strings.HasPrefix(version, "6"):
		return "6"
	case strings.HasPrefix(version, "7"):
		return "7"
	default:
		return "8"
	}
}

func logRemoteMongodbImage(taskItem *task.Task, action, database, dbName, imageTag string, info mongodbConnectionInfo) {
	message := fmt.Sprintf(
		"use local docker image %s to %s remote mongodb %s/%s via %s:%d",
		imageTag,
		action,
		database,
		dbName,
		info.Address,
		info.Port,
	)
	global.LOG.Info(message)
	if taskItem != nil {
		taskItem.Log(message)
	}
}

func logRemoteMongodbStep(taskItem *task.Task, message string) {
	global.LOG.Info(message)
	if taskItem != nil {
		taskItem.Log(message)
	}
}