fanhuadesenlinnn/cloudpods
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git synced 2026-09-20 08:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(region): server snapshot policy (#23066)

Browse Source
This commit is contained in:
屈轩
2025-08-14 10:42:30 +08:00
committed by GitHub
parent d96d1a0a02
commit c6de8a9479
17 changed files with 564 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/climc/shell/compute/snapshot_policy.go
View File

@@ -28,5 +28,7 @@ func init() {
cmd.Create(&compute.SnapshotPolicyCreateOptions{})
cmd.Perform("bind-disks", &compute.SnapshotPolicyDisksOptions{})
cmd.Perform("unbind-disks", &compute.SnapshotPolicyDisksOptions{})
cmd.Perform("bind-resources", &compute.SnapshotPolicyResourcesOptions{})
cmd.Perform("unbind-resources", &compute.SnapshotPolicyResourcesOptions{})
cmd.Perform("syncstatus", &options.BaseIdOptions{})
}

3
pkg/apis/compute/disk.go
View File

@@ -171,6 +171,9 @@ type DiskListInput struct {
// 根据虚拟机状态过滤
GuestStatus string `json:"guest_status"`
// 根据是否绑定快照策略过滤
BindingSnapshotpolicy *bool `json:"binding_snapshotpolicy"`
}
type DiskResourceInput struct {

5
pkg/apis/compute/guests.go
View File

@@ -130,8 +130,13 @@ type ServerListInput struct {
// 根据镜像发行版排序
OrderByOsDist string `json:"order_by_os_dist"`
SnapshotpolicyId string `json:"snapshotpolicy_id"`
// 是否调度到宿主机上
WithHost *bool `json:"with_host"`
// 根据是否绑定快照策略过滤
BindingSnapshotpolicy *bool `json:"binding_snapshotpolicy"`
}
func (input *ServerListInput) AfterUnmarshal() {

2
pkg/apis/compute/input.go
View File

@@ -147,6 +147,8 @@ type SnapshotPolicyListInput struct {
// 按绑定的磁盘数量排序
// pattern:asc|desc
OrderByBindDiskCount string `json:"order_by_bind_disk_count"`
// 按类型过滤
Type string `json:"type"`
}
type HostnameInput struct {

21
pkg/apis/compute/snapshotpolicy.go
View File

@@ -30,6 +30,9 @@ const (
SNAPSHOT_POLICY_APPLY_FAILED = "apply_failed"
SNAPSHOT_POLICY_CANCEL = "canceling"
SNAPSHOT_POLICY_CANCEL_FAILED = "cancel_failed"
SNAPSHOT_POLICY_TYPE_DISK = "disk"
SNAPSHOT_POLICY_TYPE_SERVER = "server"
)
type SnapshotPolicyDetails struct {
@@ -41,7 +44,8 @@ type SnapshotPolicyDetails struct {
RetentionDays int `json:"retention_days"`
BindingDiskCount int `json:"binding_disk_count"`
BindingDiskCount int `json:"binding_disk_count"`
BindingResourceCount int `json:"binding_resource_count"`
}
type SSnapshotPolicyCreateInput struct {
@@ -49,7 +53,10 @@ type SSnapshotPolicyCreateInput struct {
CloudproviderResourceInput
CloudregionResourceInput
RetentionDays int `json:"retention_days"`
RetentionDays int `json:"retention_days"`
RetentionCount int `json:"retention_count"`
// 快照类型, 目前支持 disk, server
Type string `json:"type"`
RepeatWeekdays []int `json:"repeat_weekdays"`
@@ -92,7 +99,8 @@ func (self *SSnapshotPolicyCreateInput) Validate() error {
type SSnapshotPolicyUpdateInput struct {
apis.VirtualResourceBaseUpdateInput
RetentionDays *int
RetentionDays *int
RegentionCount *int
RepeatWeekdays *[]int `json:"repeat_weekdays"`
TimePoints *[]int `json:"time_points"`
@@ -131,6 +139,13 @@ type SnapshotPolicyDisksInput struct {
Disks []string `json:"disk"`
}
type SnapshotPolicyResourcesInput struct {
Resources []struct {
Id string `json:"id"`
Type string `json:"type"`
} `json:"resources"`
}
type RepeatWeekdays []int
func (days RepeatWeekdays) String() string {

11
pkg/compute/models/disks.go
View File

@@ -210,6 +210,15 @@ func (manager *SDiskManager) ListItemFilter(
}
}
if query.BindingSnapshotpolicy != nil {
spjsq := SnapshotPolicyDiskManager.Query("disk_id").SubQuery()
if *query.BindingSnapshotpolicy {
q = q.In("id", spjsq)
} else {
q = q.NotIn("id", spjsq)
}
}
guestId := query.ServerId
if len(guestId) > 0 {
server, err := validators.ValidateModel(ctx, userCred, GuestManager, &guestId)
@@ -2852,7 +2861,7 @@ func (manager *SDiskManager) GetNeedAutoSnapshotDisks() ([]SSnapshotPolicyDisk,
}
timePoint := t.Hour()
policy := SnapshotPolicyManager.Query().Equals("cloudregion_id", api.DEFAULT_REGION_ID)
policy := SnapshotPolicyManager.Query().Equals("type", api.SNAPSHOT_POLICY_TYPE_DISK).Equals("cloudregion_id", api.DEFAULT_REGION_ID)
policy = policy.Filter(sqlchemy.Contains(policy.Field("repeat_weekdays"), fmt.Sprintf("%d", week)))
sq := policy.Filter(
sqlchemy.OR(

1
pkg/compute/models/guest_actions.go
View File

@@ -5440,6 +5440,7 @@ func (self *SGuest) validateCreateInstanceBackup(
return input, nil
}
// 创建主机快照
// 1. validate guest status, guest hypervisor
// 2. validate every disk manual snapshot count
// 3. validate snapshot quota with disk count

17
pkg/compute/models/guests.go
View File

@@ -721,6 +721,23 @@ func (manager *SGuestManager) ListItemFilter(
q = q.IsNullOrEmpty("host_id")
}
}
if len(query.SnapshotpolicyId) > 0 {
sp := SnapshotPolicyResourceManager.Query("resource_id").
Equals("resource_type", api.SNAPSHOT_POLICY_TYPE_SERVER).
Equals("snapshotpolicy_id", query.SnapshotpolicyId).SubQuery()
q = q.In("id", sp)
}
if query.BindingSnapshotpolicy != nil {
spjsq := SnapshotPolicyResourceManager.Query("resource_id").
Equals("resource_type", api.SNAPSHOT_POLICY_TYPE_SERVER).
SubQuery()
if *query.BindingSnapshotpolicy {
q = q.In("id", spjsq)
} else {
q = q.NotIn("id", spjsq)
}
}
return q, nil
}

189
pkg/compute/models/instance_snapshots.go
View File

@@ -17,6 +17,8 @@ package models
import (
"context"
"database/sql"
"fmt"
"time"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/jsonutils"
@@ -32,6 +34,7 @@ import (
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -803,3 +806,189 @@ func (self *SInstanceSnapshot) CustomizeCreate(
ownerId = guestObj.(*SGuest).GetOwnerId()
return self.SVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data)
}
func (manager *SInstanceSnapshotManager) GetNeedAutoSnapshotServers() ([]SSnapshotPolicyResource, error) {
tz, _ := time.LoadLocation(options.Options.TimeZone)
t := time.Now().In(tz)
week := t.Weekday()
if week == 0 { // sunday is zero
week += 7
}
timePoint := t.Hour()
policy := SnapshotPolicyManager.Query().Equals("type", api.SNAPSHOT_POLICY_TYPE_SERVER).Equals("cloudregion_id", api.DEFAULT_REGION_ID)
policy = policy.Filter(sqlchemy.Contains(policy.Field("repeat_weekdays"), fmt.Sprintf("%d", week)))
sq := policy.Filter(
sqlchemy.OR(
sqlchemy.Contains(policy.Field("time_points"), fmt.Sprintf(",%d,", timePoint)),
sqlchemy.Startswith(policy.Field("time_points"), fmt.Sprintf("[%d,", timePoint)),
sqlchemy.Endswith(policy.Field("time_points"), fmt.Sprintf(",%d]", timePoint)),
sqlchemy.Equals(policy.Field("time_points"), fmt.Sprintf("[%d]", timePoint)),
),
).SubQuery()
servers := GuestManager.Query().SubQuery()
q := SnapshotPolicyResourceManager.Query().Equals("resource_type", api.SNAPSHOT_POLICY_TYPE_SERVER)
q = q.Join(sq, sqlchemy.Equals(q.Field("snapshotpolicy_id"), sq.Field("id")))
q = q.Join(servers, sqlchemy.Equals(q.Field("resource_id"), servers.Field("id")))
ret := []SSnapshotPolicyResource{}
err := db.FetchModelObjects(SnapshotPolicyResourceManager, q, &ret)
if err != nil {
return nil, err
}
return ret, nil
}
func (manager *SInstanceSnapshotManager) AutoServerSnapshot(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
servers, err := manager.GetNeedAutoSnapshotServers()
if err != nil {
log.Errorf("Get auto snapshot servers id failed: %s", err)
return
}
log.Infof("auto snapshot %d servers", len(servers))
serverMap := map[string]*SGuest{}
for i := range servers {
server, err := servers[i].GetServer()
if err != nil {
log.Errorf("get server error: %v", err)
continue
}
serverMap[server.Id] = server
}
for i := range serverMap {
input := api.ServerInstanceSnapshot{}
input.GenerateName = fmt.Sprintf("auto-%s-%d", serverMap[i].Name, time.Now().Unix())
serverMap[i].PerformInstanceSnapshot(ctx, userCred, jsonutils.NewDict(), input)
}
}
var instanceSnapshotCleanupTaskRunning int32 = 0
func (manager *SInstanceSnapshotManager) CleanupInstanceSnapshots(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
if instanceSnapshotCleanupTaskRunning > 0 {
log.Errorf("Previous CleanupInstanceSnapshots tasks still running !!!")
return
}
instanceSnapshotCleanupTaskRunning = 1
defer func() {
instanceSnapshotCleanupTaskRunning = 0
}()
sq := manager.Query().Equals("status", api.INSTANCE_SNAPSHOT_READY).Startswith("name", "auto-").SubQuery()
iss := []struct {
GuestCnt int
GuestId string
}{}
q := sq.Query(
sqlchemy.COUNT("guest_cnt", sq.Field("guest_id")),
sq.Field("guest_id"),
).GroupBy(sq.Field("guest_id"))
err := q.All(&iss)
if err != nil {
log.Errorf("Cleanup instance snapshots job fetch instance snapshot failed %s", err)
return
}
guestCount := map[string]int{}
for i := range iss {
guestCount[iss[i].GuestId] = iss[i].GuestCnt
}
// cleanup retention count instance snapshots
{
sq = SnapshotPolicyManager.Query().Equals("type", api.SNAPSHOT_POLICY_TYPE_SERVER).GT("retention_count", 0).SubQuery()
spr := SnapshotPolicyResourceManager.Query().Equals("resource_type", api.SNAPSHOT_POLICY_TYPE_SERVER).SubQuery()
q = sq.Query(
sq.Field("retention_count"),
spr.Field("resource_id").Label("guest_id"),
)
q = q.Join(spr, sqlchemy.Equals(q.Field("id"), spr.Field("snapshotpolicy_id")))
guestRetentions := []struct {
GuestId string
RetentionCount int
}{}
err = q.All(&guestRetentions)
if err != nil {
log.Errorf("Cleanup instance snapshots job fetch guest retentions failed %s", err)
return
}
guestRetentionMap := map[string]int{}
for i := range guestRetentions {
if _, ok := guestRetentionMap[guestRetentions[i].GuestId]; !ok {
guestRetentionMap[guestRetentions[i].GuestId] = guestRetentions[i].RetentionCount
}
// 取最小保留个数
if guestRetentionMap[guestRetentions[i].GuestId] > guestRetentions[i].RetentionCount {
guestRetentionMap[guestRetentions[i].GuestId] = guestRetentions[i].RetentionCount
}
}
for guestId, retentionCnt := range guestRetentionMap {
if cnt, ok := guestCount[guestId]; ok && cnt > retentionCnt {
manager.startCleanupRetentionCount(ctx, userCred, guestId, cnt-retentionCnt)
}
}
}
// cleanup retention days instance snapshots
{
sq = SnapshotPolicyManager.Query().Equals("type", api.SNAPSHOT_POLICY_TYPE_SERVER).GT("retention_days", 0).SubQuery()
spr := SnapshotPolicyResourceManager.Query().Equals("resource_type", api.SNAPSHOT_POLICY_TYPE_SERVER).SubQuery()
q = sq.Query(
sq.Field("retention_days"),
spr.Field("resource_id").Label("guest_id"),
)
q = q.Join(spr, sqlchemy.Equals(q.Field("id"), spr.Field("snapshotpolicy_id")))
guestRetentions := []struct {
GuestId string
RetentionDays int
}{}
err = q.All(&guestRetentions)
if err != nil {
log.Errorf("Cleanup instance snapshots job fetch guest retentions failed %s", err)
return
}
guestRetentionMap := map[string]int{}
for i := range guestRetentions {
if _, ok := guestRetentionMap[guestRetentions[i].GuestId]; !ok {
guestRetentionMap[guestRetentions[i].GuestId] = guestRetentions[i].RetentionDays
}
// 取最小保留天数
if guestRetentionMap[guestRetentions[i].GuestId] > guestRetentions[i].RetentionDays {
guestRetentionMap[guestRetentions[i].GuestId] = guestRetentions[i].RetentionDays
}
}
for guestId, retentionDays := range guestRetentionMap {
manager.startCleanupRetentionDays(ctx, userCred, guestId, retentionDays)
}
}
}
func (manager *SInstanceSnapshotManager) startCleanupRetentionCount(ctx context.Context, userCred mcclient.TokenCredential, guestId string, cnt int) error {
q := manager.Query().Equals("guest_id", guestId).Equals("status", api.INSTANCE_SNAPSHOT_READY).Startswith("name", "auto-").Asc("created_at").Limit(cnt)
vms := []SInstanceSnapshot{}
err := db.FetchModelObjects(manager, q, &vms)
if err != nil {
return errors.Wrapf(err, "FetchModelObjects")
}
for i := range vms {
vms[i].StartInstanceSnapshotDeleteTask(ctx, userCred, "")
}
return nil
}
func (manager *SInstanceSnapshotManager) startCleanupRetentionDays(ctx context.Context, userCred mcclient.TokenCredential, guestId string, day int) error {
expiredTime := time.Now().AddDate(0, 0, -day)
q := manager.Query().Equals("guest_id", guestId).Equals("status", api.INSTANCE_SNAPSHOT_READY).Startswith("name", "auto-").LE("created_at", expiredTime)
vms := []SInstanceSnapshot{}
err := db.FetchModelObjects(manager, q, &vms)
if err != nil {
return errors.Wrapf(err, "FetchModelObjects")
}
for i := range vms {
vms[i].StartInstanceSnapshotDeleteTask(ctx, userCred, "")
}
return nil
}

77
pkg/compute/models/snapshot_policy_resources.go Normal file
View File

@@ -0,0 +1,77 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"fmt"
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
)
type SSnapshotPolicyResourceManager struct {
db.SResourceBaseManager
}
var SnapshotPolicyResourceManager *SSnapshotPolicyResourceManager
func init() {
SnapshotPolicyResourceManager = &SSnapshotPolicyResourceManager{
SResourceBaseManager: db.NewResourceBaseManager(
SSnapshotPolicyResource{},
"snapshot_policy_resources_tbl",
"snapshot_policy_resource",
"snapshot_policy_resources",
),
}
SnapshotPolicyResourceManager.SetVirtualObject(SnapshotPolicyResourceManager)
}
type SSnapshotPolicyResource struct {
db.SResourceBase
SnapshotpolicyId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
ResourceId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
ResourceType string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
}
func (man *SSnapshotPolicyResourceManager) RemoveByResource(id, typ string) error {
_, err := sqlchemy.GetDB().Exec(
fmt.Sprintf(
"delete from %s where resource_id = ? and resource_type = ?",
man.TableSpec().Name(),
), id, typ,
)
return err
}
func (self *SSnapshotPolicyResource) GetServer() (*SGuest, error) {
guest, err := GuestManager.FetchById(self.ResourceId)
if err != nil {
return nil, err
}
return guest.(*SGuest), nil
}
func (man *SSnapshotPolicyResourceManager) RemoveBySnapshotpolicy(id string) error {
_, err := sqlchemy.GetDB().Exec(
fmt.Sprintf(
"delete from %s where snapshotpolicy_id = ?",
man.TableSpec().Name(),
), id,
)
return err
}

131
pkg/compute/models/snapshotpolicy.go
View File

@@ -36,6 +36,7 @@ import (
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/logclient"
"yunion.io/x/onecloud/pkg/util/stringutils2"
)
@@ -55,12 +56,18 @@ type SSnapshotPolicy struct {
SCloudregionResourceBase `width:"36" charset:"ascii" nullable:"false" list:"domain" create:"domain_required" default:"default"`
RetentionDays int `nullable:"false" list:"user" get:"user" create:"required"`
// 快照保留天数
RetentionDays int `nullable:"false" list:"user" get:"user" update:"user" create:"required"`
// 快照保留数量, 优先级高于 RetentionDays, 且仅对本地IDC资源有效
RetentionCount int `nullable:"true" list:"user" get:"user" update:"user" create:"optional"`
// 快照类型, 目前支持 disk, server
Type string `width:"36" charset:"ascii" default:"disk" list:"user" create:"required"`
// 1~7, 1 is Monday
RepeatWeekdays api.RepeatWeekdays `charset:"utf8" create:"required" list:"user" get:"user"`
RepeatWeekdays api.RepeatWeekdays `charset:"utf8" create:"required" list:"user" get:"user" update:"user"`
// 0~23
TimePoints api.TimePoints `charset:"utf8" create:"required" list:"user" get:"user"`
TimePoints api.TimePoints `charset:"utf8" create:"required" list:"user" get:"user" update:"user"`
}
var SnapshotPolicyManager *SSnapshotPolicyManager
@@ -88,12 +95,20 @@ func (manager *SSnapshotPolicyManager) ValidateCreateData(
return nil, httperrors.NewInputParameterError("Retention days must in 1~%d or -1", options.Options.RetentionDaysLimit)
}
if input.RetentionCount > options.Options.RetentionCountLimit {
return nil, httperrors.NewInputParameterError("Retention count must less than %d", options.Options.RetentionCountLimit)
}
var err error
input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput)
if err != nil {
return nil, err
}
if len(input.Type) == 0 {
input.Type = api.SNAPSHOT_POLICY_TYPE_DISK
}
input.Status = apis.STATUS_CREATING
if len(input.CloudregionId) == 0 {
@@ -141,6 +156,11 @@ func (self *SSnapshotPolicy) ValidateUpdateData(ctx context.Context, userCred mc
return nil, httperrors.NewInputParameterError("Retention days must in 1~%d or -1", options.Options.RetentionDaysLimit)
}
}
if input.RegentionCount != nil {
if *input.RegentionCount > options.Options.RetentionCountLimit {
return nil, httperrors.NewInputParameterError("Retention count must less than %d", options.Options.RetentionCountLimit)
}
}
err = input.Validate()
if err != nil {
@@ -202,9 +222,25 @@ func (manager *SSnapshotPolicyManager) FetchCustomizeColumns(
}
pdMap[pd.SnapshotpolicyId] = append(pdMap[pd.SnapshotpolicyId], pd)
}
q = SnapshotPolicyResourceManager.Query().In("snapshotpolicy_id", policyIds)
sprs := []SSnapshotPolicyResource{}
err = q.All(&sprs)
if err != nil {
return rows
}
sprmap := map[string][]SSnapshotPolicyResource{}
for _, sp := range sprs {
_, ok := sprmap[sp.SnapshotpolicyId]
if !ok {
sprmap[sp.SnapshotpolicyId] = []SSnapshotPolicyResource{}
}
sprmap[sp.SnapshotpolicyId] = append(sprmap[sp.SnapshotpolicyId], sp)
}
for i := range rows {
res, _ := pdMap[policyIds[i]]
rows[i].BindingDiskCount = len(res)
disks := pdMap[policyIds[i]]
rows[i].BindingDiskCount = len(disks)
resources := sprmap[policyIds[i]]
rows[i].BindingResourceCount = len(resources)
}
return rows
@@ -398,6 +434,10 @@ func (sp *SSnapshotPolicy) RealDelete(ctx context.Context, userCred mcclient.Tok
if err != nil {
return errors.Wrapf(err, "delete snapshot policy disks for policy %s", sp.Name)
}
err = SnapshotPolicyResourceManager.RemoveBySnapshotpolicy(sp.Id)
if err != nil {
return errors.Wrapf(err, "delete snapshot policy resources for policy %s", sp.Name)
}
return db.DeleteModel(ctx, userCred, sp)
}
@@ -417,6 +457,9 @@ func (sp *SSnapshotPolicy) PerformBindDisks(
query jsonutils.JSONObject,
input *api.SnapshotPolicyDisksInput,
) (jsonutils.JSONObject, error) {
if sp.Type != api.SNAPSHOT_POLICY_TYPE_DISK {
return nil, httperrors.NewBadRequestError("The snapshot policy %s is not a disk snapshot policy", sp.Name)
}
if len(input.Disks) == 0 {
return nil, httperrors.NewMissingParameterError("disks")
}
@@ -450,6 +493,78 @@ func (sp *SSnapshotPolicy) PerformBindDisks(
return nil, sp.StartBindDisksTask(ctx, userCred, diskIds)
}
func (sp *SSnapshotPolicy) PerformBindResources(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
input *api.SnapshotPolicyResourcesInput,
) (jsonutils.JSONObject, error) {
if len(input.Resources) == 0 {
return nil, httperrors.NewMissingParameterError("resources")
}
for i := range input.Resources {
switch input.Resources[i].Type {
case api.SNAPSHOT_POLICY_TYPE_DISK:
_, err := validators.ValidateModel(ctx, userCred, DiskManager, &input.Resources[i].Id)
if err != nil {
return nil, err
}
case api.SNAPSHOT_POLICY_TYPE_SERVER:
_, err := validators.ValidateModel(ctx, userCred, GuestManager, &input.Resources[i].Id)
if err != nil {
return nil, err
}
default:
return nil, httperrors.NewBadRequestError("Invalid resource type: %s", input.Resources[i].Type)
}
}
for i := range input.Resources {
sr := &SSnapshotPolicyResource{}
sr.SetModelManager(SnapshotPolicyResourceManager, sr)
sr.SnapshotpolicyId = sp.Id
sr.ResourceId = input.Resources[i].Id
sr.ResourceType = input.Resources[i].Type
err := SnapshotPolicyResourceManager.TableSpec().Insert(ctx, sr)
if err != nil {
return nil, errors.Wrapf(err, "Insert")
}
}
logclient.AddActionLogWithContext(ctx, sp, logclient.ACT_BIND, input, userCred, true)
return nil, nil
}
func (sp *SSnapshotPolicy) PerformUnbindResources(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
input *api.SnapshotPolicyResourcesInput,
) (jsonutils.JSONObject, error) {
for i := range input.Resources {
switch input.Resources[i].Type {
case api.SNAPSHOT_POLICY_TYPE_DISK:
_, err := validators.ValidateModel(ctx, userCred, DiskManager, &input.Resources[i].Id)
if err != nil {
return nil, err
}
case api.SNAPSHOT_POLICY_TYPE_SERVER:
_, err := validators.ValidateModel(ctx, userCred, GuestManager, &input.Resources[i].Id)
if err != nil {
return nil, err
}
default:
return nil, httperrors.NewBadRequestError("Invalid resource type: %s", input.Resources[i].Type)
}
}
for i := range input.Resources {
err := SnapshotPolicyResourceManager.RemoveByResource(input.Resources[i].Id, input.Resources[i].Type)
if err != nil {
return nil, errors.Wrapf(err, "RemoveByResource")
}
}
logclient.AddActionLogWithContext(ctx, sp, logclient.ACT_UNBIND, input, userCred, true)
return nil, nil
}
func (sp *SSnapshotPolicy) StartUnbindDisksTask(ctx context.Context, userCred mcclient.TokenCredential, diskIds []string) error {
sp.SetStatus(ctx, userCred, api.SNAPSHOT_POLICY_CANCEL, jsonutils.Marshal(diskIds).String())
params := jsonutils.Marshal(map[string]interface{}{"disk_ids": diskIds}).(*jsonutils.JSONDict)
@@ -466,6 +581,9 @@ func (sp *SSnapshotPolicy) PerformUnbindDisks(
query jsonutils.JSONObject,
input *api.SnapshotPolicyDisksInput,
) (jsonutils.JSONObject, error) {
if sp.Type != api.SNAPSHOT_POLICY_TYPE_DISK {
return nil, httperrors.NewBadRequestError("The snapshot policy %s is not a disk snapshot policy", sp.Name)
}
if len(input.Disks) == 0 {
return nil, httperrors.NewMissingParameterError("disks")
}
@@ -524,6 +642,9 @@ func (manager *SSnapshotPolicyManager) ListItemFilter(
if err != nil {
return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.ListItemFilter")
}
if len(input.Type) > 0 {
q = q.Equals("type", input.Type)
}
return q, nil
}

71
pkg/compute/models/snapshots.go
View File

@@ -1200,6 +1200,77 @@ func (manager *SSnapshotManager) CleanupSnapshots(ctx context.Context, userCred
log.Errorf("Start snaphsot cleanup task failed %s", err)
return
}
sq := manager.Query().Equals("status", api.SNAPSHOT_READY).Equals("created_by", api.SNAPSHOT_AUTO).Equals("fake_deleted", false).SubQuery()
disks := []struct {
DiskCnt int
DiskId string
}{}
q := sq.Query(
sqlchemy.COUNT("disk_cnt", sq.Field("disk_id")),
sq.Field("disk_id"),
).GroupBy(sq.Field("disk_id"))
err = q.All(&disks)
if err != nil {
log.Errorf("Cleanup snapshots job fetch disk count failed %s", err)
return
}
diskCount := map[string]int{}
for i := range disks {
diskCount[disks[i].DiskId] = disks[i].DiskCnt
}
{
sq = SnapshotPolicyManager.Query().Equals("type", api.SNAPSHOT_POLICY_TYPE_DISK).GT("retention_count", 0).SubQuery()
spd := SnapshotPolicyDiskManager.Query().SubQuery()
q = sq.Query(
sq.Field("retention_count"),
spd.Field("disk_id"),
)
q = q.Join(spd, sqlchemy.Equals(q.Field("id"), spd.Field("snapshotpolicy_id")))
diskRetentions := []struct {
DiskId string
RetentionCount int
}{}
err = q.All(&diskRetentions)
if err != nil {
log.Errorf("Cleanup snapshots job fetch disk retentions failed %s", err)
return
}
diskRetentionMap := map[string]int{}
for i := range diskRetentions {
if _, ok := diskRetentionMap[diskRetentions[i].DiskId]; !ok {
diskRetentionMap[diskRetentions[i].DiskId] = diskRetentions[i].RetentionCount
}
// 取最小保留个数
if diskRetentionMap[diskRetentions[i].DiskId] > diskRetentions[i].RetentionCount {
diskRetentionMap[diskRetentions[i].DiskId] = diskRetentions[i].RetentionCount
}
}
for diskId, retentionCnt := range diskRetentionMap {
if cnt, ok := diskCount[diskId]; ok && cnt > retentionCnt {
manager.startCleanupRetentionCount(ctx, userCred, diskId, cnt-retentionCnt)
}
}
}
}
func (manager *SSnapshotManager) startCleanupRetentionCount(ctx context.Context, userCred mcclient.TokenCredential, diskId string, cnt int) error {
q := manager.Query().Equals("disk_id", diskId).Equals("created_by", api.SNAPSHOT_AUTO).Asc("created_at").Limit(cnt)
snapshots := []SSnapshot{}
err := db.FetchModelObjects(manager, q, &snapshots)
if err != nil {
return errors.Wrapf(err, "FetchModelObjects")
}
for i := range snapshots {
snapshots[i].StartSnapshotDeleteTask(ctx, userCred, false, "", 0, 0)
}
return nil
}
func (manager *SSnapshotManager) StartSnapshotCleanupTask(

5
pkg/compute/options/options.go
View File

@@ -108,8 +108,9 @@ type ComputeOptions struct {
AutoSnapshotHour int `default:"2" help:"What hour take sanpshot, default 02:00"`
//snapshot policy options
RetentionDaysLimit int `default:"49" help:"Days of snapshot retention, default 49 days"`
TimePointsLimit int `default:"1" help:"time point of every days, default 1 point"`
RetentionDaysLimit int `default:"49" help:"Days of snapshot retention, default 49 days"`
RetentionCountLimit int `default:"10" help:"Count of snapshot retention, default 10"`
TimePointsLimit int `default:"1" help:"time point of every days, default 1 point"`
CloudAccountBatchSyncSize int `default:"10" help:"How many cloud account syncing in a batch"`

1
pkg/compute/service/handlers.go
View File

@@ -150,6 +150,7 @@ func InitHandlers(app *appsrv.Application) {
models.InstanceSnapshotManager,
models.SnapshotManager,
models.SnapshotPolicyManager,
models.SnapshotPolicyResourceManager,
models.BaremetalagentManager,
models.LoadbalancerManager,
models.LoadbalancerListenerManager,

3
pkg/compute/service/service.go
View File

@@ -173,6 +173,7 @@ func StartServiceWithJobsAndApp(jobs func(cron *cronman.SCronJobManager), appCll
if opts.PrepaidAutoRenew {
cron.AddJobAtIntervals("AutoRenewPrepaidServers", time.Duration(opts.PrepaidAutoRenewHours)*time.Hour, models.GuestManager.AutoRenewPrepaidServer)
}
cron.AddJobAtIntervals("CleanExpiredPostpaidElasticCaches", time.Duration(opts.PrepaidExpireCheckSeconds)*time.Second, models.ElasticcacheManager.DeleteExpiredPostpaids)
cron.AddJobAtIntervals("CleanExpiredPostpaidDBInstances", time.Duration(opts.PrepaidExpireCheckSeconds)*time.Second, models.DBInstanceManager.DeleteExpiredPostpaids)
cron.AddJobAtIntervals("CleanExpiredPostpaidServers", time.Duration(opts.PrepaidExpireCheckSeconds)*time.Second, models.GuestManager.DeleteExpiredPostpaidServers)
@@ -195,7 +196,9 @@ func StartServiceWithJobsAndApp(jobs func(cron *cronman.SCronJobManager), appCll
cron.AddJobEveryFewHour("AutoPurgeSplitable", 4, 30, 0, db.AutoPurgeSplitable, false)
cron.AddJobEveryFewHour("AutoDiskSnapshot", 1, 5, 0, models.DiskManager.AutoDiskSnapshot, false)
cron.AddJobEveryFewHour("AutoServerSnapshot", 1, 5, 0, models.InstanceSnapshotManager.AutoServerSnapshot, false)
cron.AddJobEveryFewHour("SnapshotsCleanup", 1, 35, 0, models.SnapshotManager.CleanupSnapshots, false)
cron.AddJobEveryFewHour("InstanceSnapshotsCleanup", 1, 35, 0, models.InstanceSnapshotManager.CleanupInstanceSnapshots, false)
cron.AddJobEveryFewHour("AutoCleanImageCache", 1, 5, 0, models.CachedimageManager.AutoCleanImageCaches, false)

3
pkg/mcclient/options/compute/servers.go
View File

@@ -79,7 +79,8 @@ type ServerListOptions struct {
WithUserMeta *bool `help:"filter by user metadata" negative:"without_user_meta"`
WithHost *bool `help:"filter guest with host or not" negative:"without_host"`
WithHost *bool `help:"filter guest with host or not" negative:"without_host"`
SnapshotpolicyId string `help:"filter guest with snapshotpolicy or not" json:"snapshotpolicy_id"`
}
func (o *ServerListOptions) Params() (jsonutils.JSONObject, error) {

37
pkg/mcclient/options/compute/snapshotpolicy.go
View File

@@ -15,8 +15,11 @@
package compute
import (
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient/options"
)
@@ -35,9 +38,11 @@ type SnapshotPolicyCreateOptions struct {
CloudregionId string
ManagerId string
RetentionDays int `help:"snapshot retention days" default:"-1"`
RepeatWeekdays []int `help:"snapshot create days on week"`
TimePoints []int `help:"snapshot create time points on one day"`
Type string `help:"snapshot type" default:"disk" choices:"disk|server"`
RetentionDays int `help:"snapshot retention days" default:"-1"`
RetentionCount int `help:"snapshot retention count" default:"-1"`
RepeatWeekdays []int `help:"snapshot create days on week"`
TimePoints []int `help:"snapshot create time points on one day"`
}
func (opts *SnapshotPolicyCreateOptions) Params() (jsonutils.JSONObject, error) {
@@ -52,3 +57,29 @@ type SnapshotPolicyDisksOptions struct {
func (opts *SnapshotPolicyDisksOptions) Params() (jsonutils.JSONObject, error) {
return jsonutils.Marshal(map[string]interface{}{"disks": opts.Disks}), nil
}
type SnapshotPolicyResourcesOptions struct {
options.BaseIdOptions
Resources []string `help:"resource info etc: disk:id,server:id"`
}
func (opts *SnapshotPolicyResourcesOptions) Params() (jsonutils.JSONObject, error) {
resources := []struct {
Id string `json:"id"`
Type string `json:"type"`
}{}
for _, resource := range opts.Resources {
parts := strings.SplitN(resource, ":", 2)
if len(parts) != 2 {
return nil, httperrors.NewBadRequestError("Invalid resource info: %s", resource)
}
resources = append(resources, struct {
Id string `json:"id"`
Type string `json:"type"`
}{
Id: parts[1],
Type: parts[0],
})
}
return jsonutils.Marshal(map[string]interface{}{"resources": resources}), nil
}