fanhuadesenlinnn/cloudpods
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git synced 2026-09-20 08:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support pod container (#19706)

Browse Source 
* feat(region,scheduler,host): support pod and container

* feat(region,host): container nvidia gpu support

* fix(region,host): function parameters

---------

Co-authored-by: wanyaoqi <d3lx.yq@gmail.com>
This commit is contained in:
Zexi Li
2024-03-12 18:47:44 +08:00
committed by GitHub
parent cf3d468814
commit 6313ce87a3
114 changed files with 43182 additions and 827 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
cmd/climc/shell/compute/containers.go Normal file
View File

@@ -0,0 +1,30 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
import (
"yunion.io/x/onecloud/cmd/climc/shell"
modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
options "yunion.io/x/onecloud/pkg/mcclient/options/compute"
)
func init() {
cmd := shell.NewResourceCmd(&modules.Containers)
cmd.Create(new(options.ContainerCreateOptions))
cmd.List(new(options.ContainerListOptions))
cmd.BatchDelete(new(options.ContainerDeleteOptions))
cmd.BatchPerform("stop", new(options.ContainerStopOptions))
cmd.BatchPerform("start", new(options.ContainerStartOptions))
}

41
cmd/climc/shell/compute/server_pod.go Normal file
View File

@@ -0,0 +1,41 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
import (
"yunion.io/x/onecloud/pkg/mcclient"
modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
options "yunion.io/x/onecloud/pkg/mcclient/options/compute"
)
func init() {
R(&options.PodCreateOptions{}, "pod-create", "Create a container pod", func(s *mcclient.ClientSession, opts *options.PodCreateOptions) error {
params, err := opts.Params()
if err != nil {
return err
}
if opts.Count > 1 {
results := modules.Servers.BatchCreate(s, params.JSON(params), opts.Count)
printBatchResults(results, modules.Servers.GetColumns(s))
} else {
server, err := modules.Servers.Create(s, params.JSON(params))
if err != nil {
return err
}
printObject(server)
}
return nil
})
}

2
cmd/climc/shell/compute/specs.go
View File

@@ -131,7 +131,7 @@ func init() {
return err
}
data := &printutils.ListResult{Data: hosts, Total: len(hosts)}
printList(data, []string{"ID", "Name", "Mem", "CPU", "Storage_Info"})
printList(data, []string{"ID", "Name", "MEM", "CPU", "Storage_Info"})
return nil
})

104
cmd/run-pod/main.go Normal file
View File

@@ -0,0 +1,104 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"time"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/util/pod"
)
func main() {
ctl, err := pod.NewCRI("unix:///var/run/onecloud/containerd/containerd.sock", 3*time.Second)
if err != nil {
log.Fatalf("NewCRI: %v", err)
}
ctx := context.Background()
imgs, err := ctl.ListImages(ctx, nil)
if err != nil {
log.Fatalf("ListImages: %v", err)
}
for _, img := range imgs {
log.Infof("get img: %s", img.String())
}
ver, err := ctl.Version(context.Background())
if err != nil {
log.Fatalf("get version: %v", err)
}
log.Infof("get version: %s", ver.String())
// create container
podCfg := &runtimeapi.PodSandboxConfig{
Metadata: &runtimeapi.PodSandboxMetadata{
Name: "test-gpu",
Uid: "e25e38ef-fe98-4993-8641-699cd0530fc0",
Namespace: "27c9464ab54947328a29298761895be3",
Attempt: 1,
},
Hostname: "test-gpu",
LogDirectory: "",
DnsConfig: nil,
PortMappings: nil,
Labels: nil,
Annotations: nil,
Linux: nil,
Windows: nil,
}
ctrCfgs := []*runtimeapi.ContainerConfig{
{
Metadata: &runtimeapi.ContainerMetadata{
Name: "nvidia-smi",
},
Image: &runtimeapi.ImageSpec{
Image: "ubuntu",
},
Command: []string{"sleep", "100d"},
Linux: &runtimeapi.LinuxContainerConfig{
//SecurityContext: &runtimeapi.LinuxContainerSecurityContext{
// Privileged: true,
//},
},
Envs: []*runtimeapi.KeyValue{
{
Key: "NVIDIA_VISIBLE_DEVICES",
Value: "GPU-e588f4f5-29a4-4374-a335-86e120b50e14,GPU-f7160578-ba3b-3e42-6991-14c815ce032a,GPU-679b381b-eb98-62b7-c7c4-175f7d751aad",
},
{
Key: "NVIDIA_DRIVER_CAPABILITIES",
Value: "compute,utility",
},
},
/*Devices: []*runtimeapi.Device{
{
HostPath: "/dev/nvidia0",
ContainerPath: "/dev/nvidia0",
Permissions: "rwm",
},
},*/
},
}
resp, err := ctl.RunContainers(ctx, podCfg, ctrCfgs, "")
if err != nil {
log.Fatalf("RunContainers: %v", err)
}
log.Infof("RunContainers: %s", jsonutils.Marshal(resp))
}

2
go.mod
View File

@@ -87,6 +87,7 @@ require (
k8s.io/apimachinery v0.19.3
k8s.io/client-go v0.19.3
k8s.io/cluster-bootstrap v0.19.3
k8s.io/cri-api v0.22.17
moul.io/http2curl/v2 v2.3.0
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240308095621-979378c55149
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32
@@ -248,6 +249,7 @@ require (
github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect
github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/zexi/golosetup v0.0.1 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
go.opencensus.io v0.22.4 // indirect
go.uber.org/atomic v1.7.0 // indirect

10
go.sum
View File

@@ -566,6 +566,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -752,6 +753,10 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zexi/golosetup v0.0.0-20181117053200-8c308e8bbf44 h1:t8QkjgKJTQyJT7/MJJgAdRGt316wkQ0Ar3poFd1E0X0=
github.com/zexi/golosetup v0.0.0-20181117053200-8c308e8bbf44/go.mod h1:uswjAAGley+FRw3bgWor83twfO8Ru6owCpic+Xnh2Xo=
github.com/zexi/golosetup v0.0.1 h1:y7RRI/2xqzR2ByO3a2QTARUp5ivbC5R6zFXEx2WrInA=
github.com/zexi/golosetup v0.0.1/go.mod h1:U4bxWs+J/Kq1/pyRHBCOk2FzqO+AYA4GTzrwgeDmWvY=
github.com/zexi/influxql-to-metricsql v0.0.6 h1:E16T4oqgjIJtSNVvhGGHnw+pmY3yGz2iRsmmmSVuOpY=
github.com/zexi/influxql-to-metricsql v0.0.6/go.mod h1:PyRRM+3Zrzzig6J4okYLeSv+/d+5GaL5ccBaUKQABNs=
go.etcd.io/etcd/api/v3 v3.5.0 h1:GsV3S+OfZEOCNXdtNkBSR7kgLobAa/SO6tCxRa0GAYw=
@@ -867,6 +872,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
@@ -950,6 +956,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1140,6 +1147,7 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@@ -1187,6 +1195,8 @@ k8s.io/client-go v0.19.3 h1:ctqR1nQ52NUs6LpI0w+a5U+xjYwflFwA13OJKcicMxg=
k8s.io/client-go v0.19.3/go.mod h1:+eEMktZM+MG0KO+PTkci8xnbCZHvj9TqR6Q1XDUIJOM=
k8s.io/cluster-bootstrap v0.19.3 h1:LIVoRLTjJt3Yq/u+cV8h3cxGPwz9w+SFXtHdiKxfD7A=
k8s.io/cluster-bootstrap v0.19.3/go.mod h1:yZPVza5jZABN+xe4y6By4DnZI+UF9TxYj80etnDzZ5w=
k8s.io/cri-api v0.22.17 h1:lUY0gpQXQl1hFDMbuYb+eB94Vu/8QoLKYCb8ACnpAHE=
k8s.io/cri-api v0.22.17/go.mod h1:uAw9CICQq20/1yB4ZnWT2TjJyMMROl4typFfWaURLwQ=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A=

2
pkg/apis/compute/api.go
View File

@@ -627,6 +627,8 @@ type ServerCreateInput struct {
// 指定用于新建主机的主机镜像ID
GuestImageID string `json:"guest_image_id"`
Pod *PodCreateInput `json:"pod"`
}
func (input *ServerCreateInput) AfterUnmarshal() {

134
pkg/apis/compute/container.go Normal file
View File

@@ -0,0 +1,134 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
import (
"reflect"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/gotypes"
"yunion.io/x/onecloud/pkg/apis"
)
func init() {
gotypes.RegisterSerializable(reflect.TypeOf(new(ContainerSpec)), func() gotypes.ISerializable {
return new(ContainerSpec)
})
}
const (
CONTAINER_DEV_CPH_AMD_GPU = "CPH_AMD_GPU"
CONTAINER_DEV_CPH_AOSP_BINDER = "CPH_AOSP_BINDER"
CONTAINER_DEV_NETINT_CA_ASIC = "NETINT_CA_ASIC"
CONTAINER_DEV_NETINT_CA_QUADRA = "NETINT_CA_QUADRA"
CONTAINER_DEV_NVIDIA_GPU = "NVIDIA_GPU"
)
const (
CONTAINER_STORAGE_LOCAL_RAW = "local_raw"
)
const (
CONTAINER_STATUS_PULLING_IMAGE = "pulling_image"
CONTAINER_STATUS_PULL_IMAGE_FAILED = "pull_image_failed"
CONTAINER_STATUS_PULLED_IMAGE = "pulled_image"
CONTAINER_STATUS_CREATING = "creating"
CONTAINER_STATUS_CREATE_FAILED = "create_failed"
CONTAINER_STATUS_STARTING = "starting"
CONTAINER_STATUS_START_FAILED = "start_failed"
CONTAINER_STATUS_STOPPING = "stopping"
CONTAINER_STATUS_STOP_FAILED = "stop_failed"
CONTAINER_STATUS_SYNC_STATUS = "sync_status"
CONTAINER_STATUS_SYNC_STATUS_FAILED = "sync_status_failed"
CONTAINER_STATUS_UNKNOWN = "unknown"
CONTAINER_STATUS_CREATED = "created"
CONTAINER_STATUS_EXITED = "exited"
CONTAINER_STATUS_RUNNING = "running"
CONTAINER_STATUS_DELETING = "deleting"
CONTAINER_STATUS_DELETE_FAILED = "delete_failed"
)
const (
CONTAINER_METADATA_CRI_ID = "cri_id"
)
type ContainerSpec struct {
apis.ContainerSpec
// Mounts for the container.
// Mounts []*ContainerMount `json:"mounts"`
Devices []*ContainerDevice `json:"devices"`
}
func (c *ContainerSpec) String() string {
return jsonutils.Marshal(c).String()
}
func (c *ContainerSpec) IsZero() bool {
if reflect.DeepEqual(*c, ContainerSpec{}) {
return true
}
return false
}
type ContainerCreateInput struct {
apis.VirtualResourceCreateInput
GuestId string `json:"guest_id"`
Spec ContainerSpec `json:"spec"`
// swagger:ignore
SkipTask bool `json:"skip_task"`
}
type ContainerListInput struct {
apis.VirtualResourceListInput
}
type ContainerStopInput struct {
Timeout int `json:"timeout"`
}
type ContainerSyncStatusResponse struct {
Status string `json:"status"`
}
type ContainerDesc struct {
Id string `json:"id"`
Name string `json:"name"`
Spec *ContainerSpec `json:"spec"`
}
type ContainerHostDevice struct {
// Path of the device within the container.
ContainerPath string `json:"container_path"`
// Path of the device on the host.
HostPath string `json:"host_path"`
// Cgroups permissions of the device, candidates are one or more of
// * r - allows container to read from the specified device.
// * w - allows container to write to the specified device.
// * m - allows container to create device files that do not yet exist.
Permissions string `json:"permissions"`
}
type ContainerIsolatedDevice struct {
Index *int `json:"index"`
Id string `json:"id"`
}
type ContainerDevice struct {
Type apis.ContainerDeviceType `json:"type"`
IsolatedDevice *ContainerIsolatedDevice `json:"isolated_device"`
Host *ContainerHostDevice `json:"host"`
}

10
pkg/apis/compute/guest_const.go
View File

@@ -182,7 +182,7 @@ const (
SHUTDOWN_TERMINATE = "terminate"
HYPERVISOR_KVM = "kvm"
HYPERVISOR_CONTAINER = "container"
HYPERVISOR_POD = "pod"
HYPERVISOR_BAREMETAL = "baremetal"
HYPERVISOR_ESXI = compute.HYPERVISOR_ESXI
HYPERVISOR_HYPERV = "hyperv"
@@ -266,7 +266,7 @@ var HYPERVISORS = []string{
HYPERVISOR_KVM,
HYPERVISOR_BAREMETAL,
HYPERVISOR_ESXI,
HYPERVISOR_CONTAINER,
HYPERVISOR_POD,
HYPERVISOR_ALIYUN,
HYPERVISOR_APSARA,
HYPERVISOR_AZURE,
@@ -301,7 +301,7 @@ var HYPERVISORS = []string{
var ONECLOUD_HYPERVISORS = []string{
HYPERVISOR_BAREMETAL,
HYPERVISOR_KVM,
HYPERVISOR_CONTAINER,
HYPERVISOR_POD,
}
var PUBLIC_CLOUD_HYPERVISORS = []string{
@@ -345,7 +345,7 @@ var HYPERVISOR_HOSTTYPE = map[string]string{
HYPERVISOR_KVM: HOST_TYPE_HYPERVISOR,
HYPERVISOR_BAREMETAL: HOST_TYPE_BAREMETAL,
HYPERVISOR_ESXI: HOST_TYPE_ESXI,
HYPERVISOR_CONTAINER: HOST_TYPE_KUBELET,
HYPERVISOR_POD: HOST_TYPE_CONTAINER,
HYPERVISOR_ALIYUN: HOST_TYPE_ALIYUN,
HYPERVISOR_APSARA: HOST_TYPE_APSARA,
HYPERVISOR_AZURE: HOST_TYPE_AZURE,
@@ -381,7 +381,7 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
HOST_TYPE_HYPERVISOR: HYPERVISOR_KVM,
HOST_TYPE_BAREMETAL: HYPERVISOR_BAREMETAL,
HOST_TYPE_ESXI: HYPERVISOR_ESXI,
HOST_TYPE_KUBELET: HYPERVISOR_CONTAINER,
HOST_TYPE_CONTAINER: HYPERVISOR_POD,
HOST_TYPE_ALIYUN: HYPERVISOR_ALIYUN,
HOST_TYPE_APSARA: HYPERVISOR_APSARA,
HOST_TYPE_AZURE: HYPERVISOR_AZURE,

3
pkg/apis/compute/guests.go
View File

@@ -905,6 +905,9 @@ type GuestJsonDesc struct {
IsDaemon bool `json:"is_daemon"`
LightMode bool `json:"light_mode"`
Hypervisor string `json:"hypervisor"`
Containers []*ContainerDesc `json:"containers"`
}
type ServerSetBootIndexInput struct {

4
pkg/apis/compute/host_const.go
View File

@@ -21,9 +21,9 @@ import (
const (
HOST_TYPE_BAREMETAL = "baremetal"
HOST_TYPE_HYPERVISOR = "hypervisor" // KVM
HOST_TYPE_CONTAINER = "container"
HOST_TYPE_KVM = "kvm"
HOST_TYPE_ESXI = compute.HOST_TYPE_ESXI // # VMWare vSphere ESXi
HOST_TYPE_KUBELET = "kubelet" // # Kubernetes Kubelet
HOST_TYPE_HYPERV = "hyperv" // # Microsoft Hyper-V
HOST_TYPE_XEN = "xen" // # XenServer
@@ -123,7 +123,7 @@ var HOST_TYPES = []string{
HOST_TYPE_BAREMETAL,
HOST_TYPE_HYPERVISOR,
HOST_TYPE_ESXI,
HOST_TYPE_KUBELET,
HOST_TYPE_CONTAINER,
HOST_TYPE_XEN,
HOST_TYPE_ALIYUN,
HOST_TYPE_APSARA,

4
pkg/apis/compute/isolated_device.go
View File

@@ -93,6 +93,8 @@ type IsolatedDeviceCreateInput struct {
VendorDeviceId string `json:"vendor_device_id"`
// PCIE information
PCIEInfo *IsolatedDevicePCIEInfo `json:"pcie_info"`
// Host device path
DevicePath string `json:"device_path"`
}
type IsolatedDeviceReservedResourceInput struct {
@@ -112,6 +114,8 @@ type IsolatedDeviceUpdateInput struct {
DevType string `json:"dev_type"`
// PCIE information
PCIEInfo *IsolatedDevicePCIEInfo `json:"pcie_info"`
// Host device path
DevicePath string `json:"device_path"`
}
type IsolatedDeviceJsonDesc struct {

7
pkg/apis/compute/isolated_device_const.go
View File

@@ -33,6 +33,8 @@ const MEAT_PROBED_HOST_COUNT = "probed_host_count"
var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE}
var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE}
var VALID_CONTAINER_DEVICE_TYPES = []string{CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA, CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU}
var VALID_PASSTHROUGH_TYPES = []string{
DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE,
GPU_VGA_TYPE, NVME_PT_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE,
@@ -53,3 +55,8 @@ const (
ISOLATED_DEVICE_MODEL_METADATA_MEMORY_MB = "memory_mb"
ISOLATED_DEVICE_MODEL_METADATA_TFLOPS = "tflops"
)
func init() {
VALID_PASSTHROUGH_TYPES = append(VALID_PASSTHROUGH_TYPES, VALID_CONTAINER_DEVICE_TYPES...)
VALID_ATTACH_TYPES = append(VALID_ATTACH_TYPES, VALID_CONTAINER_DEVICE_TYPES...)
}

58
pkg/apis/compute/pod.go Normal file
View File

@@ -0,0 +1,58 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
const (
POD_STATUS_CREATING_CONTAINER = "creating_container"
POD_STATUS_CREATE_CONTAINER_FAILED = "create_container_failed"
POD_STATUS_DELETING_CONTAINER = "deleting_container"
POD_STATUS_DELETE_CONTAINER_FAILED = "delete_container_failed"
)
const (
POD_METADATA_CRI_ID = "cri_id"
POD_METADATA_CRI_CONFIG = "cri_config"
)
type PodContainerCreateInput struct {
// Container name
Name string `json:"name"`
ContainerSpec
}
type PodPortMappingProtocol string
const (
PodPortMappingProtocolTCP = "tcp"
PodPortMappingProtocolUDP = "udp"
PodPortMappingProtocolSCTP = "sctp"
)
type PodPortMapping struct {
Protocol PodPortMappingProtocol `json:"protocol"`
ContainerPort int32 `json:"container_port"`
HostPort int32 `json:"host_port"`
HostIp string `json:"host_ip"`
}
type PodCreateInput struct {
Containers []*PodContainerCreateInput `json:"containers"`
PortMappings []*PodPortMapping `json:"port_mappings"`
}
type PodStartResponse struct {
CRIId string `json:"cri_id"`
IsRunning bool `json:"is_running"`
}

105
pkg/apis/container.go Normal file
View File

@@ -0,0 +1,105 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package apis
import "yunion.io/x/pkg/util/sets"
type ContainerKeyValue struct {
Key string `json:"key"`
Value string `json:"value"`
}
type ContainerSpec struct {
// Image to use.
Image string `json:"image"`
// Image pull policy
ImagePullPolicy ImagePullPolicy `json:"image_pull_policy"`
// Command to execute (i.e., entrypoint for docker)
Command []string `json:"command"`
// Args for the Command (i.e. command for docker)
Args []string `json:"args"`
// Current working directory of the command.
WorkingDir string `json:"working_dir"`
// List of environment variable to set in the container.
Envs []*ContainerKeyValue `json:"envs"`
// Enable lxcfs
EnableLxcfs bool `json:"enable_lxcfs"`
// Volume mounts
VolumeMounts []*ContainerVolumeMount `json:"volume_mounts"`
}
type ImagePullPolicy string
const (
ImagePullPolicyAlways = "Always"
ImagePullPolicyIfNotPresent = "IfNotPresent"
)
type ContainerVolumeMountType string
const (
CONTAINER_VOLUME_MOUNT_TYPE_DISK ContainerVolumeMountType = "disk"
CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH ContainerVolumeMountType = "host_path"
)
type ContainerDeviceType string
const (
CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE ContainerDeviceType = "isolated_device"
CONTAINER_DEVICE_TYPE_HOST ContainerDeviceType = "host"
)
type ContainerMountPropagation string
const (
// No mount propagation ("private" in Linux terminology).
MOUNTPROPAGATION_PROPAGATION_PRIVATE ContainerMountPropagation = "private"
// Mounts get propagated from the host to the container ("rslave" in Linux).
MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER ContainerMountPropagation = "rslave"
// Mounts get propagated from the host to the container and from the
// container to the host ("rshared" in Linux).
MOUNTPROPAGATION_PROPAGATION_BIDIRECTIONAL ContainerMountPropagation = "rshared"
)
var (
ContainerMountPropagations = sets.NewString(
string(MOUNTPROPAGATION_PROPAGATION_PRIVATE), string(MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER), string(MOUNTPROPAGATION_PROPAGATION_BIDIRECTIONAL))
)
type ContainerVolumeMount struct {
Type ContainerVolumeMountType `json:"type"`
Disk *ContainerVolumeMountDisk `json:"disk"`
HostPath *ContainerVolumeMountHostPath `json:"host_path"`
// Mounted read-only if true, read-write otherwise (false or unspecified).
ReadOnly bool `json:"read_only"`
// Path within the container at which the volume should be mounted. Must
// not contain ':'.
MountPath string `json:"mount_path"`
// If set, the mount needs SELinux relabeling.
SelinuxRelabel bool `json:"selinux_relabel,omitempty"`
// Requested propagation mode.
Propagation ContainerMountPropagation `json:"propagation,omitempty"`
}
type ContainerVolumeMountDisk struct {
Index *int `json:"index,omitempty"`
Id string `json:"id"`
SubDirectory string `json:"sub_directory"`
StorageSizeFile string `json:"storage_size_file"`
}
type ContainerVolumeMountHostPath struct {
Path string `json:"path"`
}

71
pkg/apis/host/container.go Normal file
View File

@@ -0,0 +1,71 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package host
import "yunion.io/x/onecloud/pkg/apis"
type ContainerSpec struct {
apis.ContainerSpec
Devices []*ContainerDevice `json:"devices"`
}
type ContainerDevice struct {
Type apis.ContainerDeviceType `json:"type"`
ContainerPath string `json:"container_path"`
Permissions string `json:"permissions"`
IsolatedDevice *ContainerIsolatedDevice `json:"isolated_device"`
Host *ContainerHostDevice `json:"host"`
Disk *ContainerDiskDevice `json:"disk"`
}
type ContainerIsolatedDevice struct {
Id string `json:"id"`
Addr string `json:"addr"`
Path string `json:"path"`
DeviceType string `json:"device_type"`
}
type ContainerHostDevice struct {
// Path of the device on the host.
HostPath string `json:"host_path"`
}
type ContainerDiskDevice struct {
Id string `json:"id"`
}
type ContainerCreateInput struct {
Name string `json:"name"`
GuestId string `json:"guest_id"`
Spec *ContainerSpec `json:"spec"`
}
type ContainerPullImageAuthConfig struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Auth string `json:"auth,omitempty"`
ServerAddress string `json:"server_address,omitempty"`
// IdentityToken is used to authenticate the user and get
// an access token for the registry.
IdentityToken string `json:"identity_token,omitempty"`
// RegistryToken is a bearer token to be sent to a registry
RegistryToken string `json:"registry_token,omitempty"`
}
type ContainerPullImageInput struct {
Image string `json:"image"`
PullPolicy apis.ImagePullPolicy `json:"pull_policy"`
Auth *ContainerPullImageAuthConfig `json:"auth"`
}

1
pkg/compute/container_drivers/device/doc.go Normal file
View File

@@ -0,0 +1 @@
package device // import "yunion.io/x/onecloud/pkg/compute/container_drivers/device"

66
pkg/compute/container_drivers/device/host.go Normal file
View File

@@ -0,0 +1,66 @@
package device
import (
"context"
"strings"
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
func init() {
models.RegisterContainerDeviceDriver(newHostDevice())
}
type hostDevice struct {
}
func newHostDevice() models.IContainerDeviceDriver {
return &hostDevice{}
}
func (h hostDevice) GetType() apis.ContainerDeviceType {
return apis.CONTAINER_DEVICE_TYPE_HOST
}
func (h hostDevice) ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, dev *api.ContainerDevice, input *api.ServerCreateInput) error {
_, err := h.ValidateCreateData(ctx, userCred, nil, dev)
return err
}
func (h hostDevice) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *models.SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) {
host := dev.Host
if host == nil {
return nil, httperrors.NewNotEmptyError("host is nil")
}
if host.HostPath == "" {
return nil, httperrors.NewNotEmptyError("host_path is empty")
}
if host.ContainerPath == "" {
return nil, httperrors.NewNotEmptyError("container_path is empty")
}
if host.Permissions == "" {
return nil, httperrors.NewNotEmptyError("permissions is empty")
}
for _, p := range strings.Split(host.Permissions, "") {
switch p {
case "r", "w", "m":
default:
return nil, httperrors.NewInputParameterError("wrong permission %s", p)
}
}
return dev, nil
}
func (h hostDevice) ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error) {
return &hostapi.ContainerDevice{
Type: apis.CONTAINER_DEVICE_TYPE_HOST,
ContainerPath: dev.Host.ContainerPath,
Permissions: dev.Host.Permissions,
Host: &hostapi.ContainerHostDevice{HostPath: dev.Host.HostPath},
}, nil
}

123
pkg/compute/container_drivers/device/isolated_device.go Normal file
View File

@@ -0,0 +1,123 @@
package device
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/sets"
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
func init() {
models.RegisterContainerDeviceDriver(newIsolatedDevice())
}
type isolatedDevice struct{}
func newIsolatedDevice() models.IContainerDeviceDriver {
return &isolatedDevice{}
}
func (i isolatedDevice) GetType() apis.ContainerDeviceType {
return apis.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE
}
func (i isolatedDevice) validateCreateData(dev *api.ContainerDevice) error {
isoDev := dev.IsolatedDevice
if isoDev == nil {
return httperrors.NewNotEmptyError("isolated_device is nil")
}
if isoDev.Index == nil && isoDev.Id == "" {
return httperrors.NewNotEmptyError("one of index or id is required")
}
if isoDev.Index != nil {
if *isoDev.Index < 0 {
return httperrors.NewInputParameterError("index is less than 0")
}
}
return nil
}
func (i isolatedDevice) ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, dev *api.ContainerDevice, input *api.ServerCreateInput) error {
if err := i.validateCreateData(dev); err != nil {
return errors.Wrapf(err, "validate create data %s", jsonutils.Marshal(dev))
}
isoDev := dev.IsolatedDevice
if isoDev.Id != "" {
return httperrors.NewInputParameterError("can't specify id %s when creating pod", isoDev.Id)
}
if isoDev.Index == nil {
return httperrors.NewNotEmptyError("index is required")
}
inputDevs := input.IsolatedDevices
if *isoDev.Index >= len(inputDevs) {
return httperrors.NewInputParameterError("disk.index %d is large than disk size %d", isoDev.Index, len(inputDevs))
}
return nil
}
func (i isolatedDevice) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *models.SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) {
if err := i.validateCreateData(dev); err != nil {
return nil, errors.Wrapf(err, "validate create data %s", jsonutils.Marshal(dev))
}
isoDev := dev.IsolatedDevice
podDevs, err := pod.GetIsolatedDevices()
if err != nil {
return nil, errors.Wrap(err, "get isolated devices")
}
if isoDev.Index != nil {
index := *isoDev.Index
if index >= len(podDevs) {
return nil, httperrors.NewInputParameterError("index %d is large than isolated device size %d", index, len(podDevs))
}
isoDev.Id = podDevs[index].GetId()
// remove index
isoDev.Index = nil
} else {
if isoDev.Id == "" {
return nil, httperrors.NewNotEmptyError("id is empty")
}
foundDisk := false
for _, d := range podDevs {
if d.GetId() == isoDev.Id || d.GetName() == isoDev.Id {
isoDev.Id = d.GetId()
foundDisk = true
devType := d.DevType
if !sets.NewString(api.VALID_CONTAINER_DEVICE_TYPES...).Has(devType) {
return nil, httperrors.NewInputParameterError("device type %s is not supported by container", devType)
}
break
}
}
if !foundDisk {
return nil, httperrors.NewNotFoundError("not found pod device by %s", isoDev.Id)
}
}
dev.IsolatedDevice = isoDev
return dev, nil
}
func (i isolatedDevice) ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error) {
input := dev.IsolatedDevice
isoDevObj, err := models.IsolatedDeviceManager.FetchById(input.Id)
if err != nil {
return nil, errors.Wrapf(err, "Fetch isolated device by id %s", input.Id)
}
isoDev := isoDevObj.(*models.SIsolatedDevice)
return &hostapi.ContainerDevice{
Type: dev.Type,
IsolatedDevice: &hostapi.ContainerIsolatedDevice{
Id: isoDev.GetId(),
Addr: isoDev.Addr,
Path: isoDev.DevicePath,
DeviceType: isoDev.DevType,
},
}, nil
}

101
pkg/compute/container_drivers/volume_mount/disk.go Normal file
View File

@@ -0,0 +1,101 @@
package volume_mount
import (
"context"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
func init() {
models.RegisterContainerVolumeMountDriver(newDisk())
}
type disk struct{}
func newDisk() models.IContainerVolumeMountDriver {
return &disk{}
}
func (d disk) GetType() apis.ContainerVolumeMountType {
return apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK
}
func (d disk) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) {
disk := vm.Disk
if disk == nil {
return nil, httperrors.NewNotEmptyError("disk is nil")
}
if disk.Index == nil && disk.Id == "" {
return nil, httperrors.NewNotEmptyError("one of index or id is required")
}
if disk.Index != nil {
if *disk.Index < 0 {
return nil, httperrors.NewInputParameterError("index is less than 0")
}
}
return vm, nil
}
func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *models.SGuest, vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) {
if _, err := d.validateCreateData(ctx, userCred, vm); err != nil {
return nil, err
}
disks, err := pod.GetDisks()
if err != nil {
return nil, errors.Wrap(err, "get pod disks")
}
disk := vm.Disk
if disk.Index != nil {
diskIndex := *disk.Index
if diskIndex >= len(disks) {
return nil, httperrors.NewInputParameterError("disk.index %d is large than disk size %d", diskIndex, len(disks))
}
vm.Disk.Id = disks[diskIndex].GetId()
// remove index
vm.Disk.Index = nil
} else {
if disk.Id == "" {
return nil, httperrors.NewNotEmptyError("disk.id is empty")
}
foundDisk := false
for _, d := range disks {
if d.GetId() == disk.Id || d.GetName() == disk.Id {
disk.Id = d.GetId()
foundDisk = true
break
}
}
if !foundDisk {
return nil, httperrors.NewNotFoundError("not found pod disk by %s", disk.Id)
}
}
return vm, nil
}
func (d disk) ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount, input *api.ServerCreateInput) error {
if _, err := d.validateCreateData(ctx, userCred, vm); err != nil {
return err
}
disk := vm.Disk
if disk.Id != "" {
return httperrors.NewInputParameterError("can't specify disk_id %s when creating pod", disk.Id)
}
if disk.Index == nil {
return httperrors.NewNotEmptyError("disk.index is required")
}
diskIndex := *disk.Index
disks := input.Disks
if diskIndex < 0 {
return httperrors.NewInputParameterError("disk.index %d is less than 0", diskIndex)
}
if diskIndex >= len(disks) {
return httperrors.NewInputParameterError("disk.index %d is large than disk size %d", diskIndex, len(disks))
}
return nil
}

1
pkg/compute/container_drivers/volume_mount/doc.go Normal file
View File

@@ -0,0 +1 @@
package volume_mount // import "yunion.io/x/onecloud/pkg/compute/container_drivers/volume_mount"

40
pkg/compute/container_drivers/volume_mount/host_local.go Normal file
View File

@@ -0,0 +1,40 @@
package volume_mount
import (
"context"
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
func init() {
models.RegisterContainerVolumeMountDriver(newHostLocal())
}
type hostLocal struct{}
func newHostLocal() models.IContainerVolumeMountDriver {
return &hostLocal{}
}
func (h hostLocal) GetType() apis.ContainerVolumeMountType {
return apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH
}
func (h hostLocal) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *models.SGuest, vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) {
return vm, h.ValidatePodCreateData(ctx, userCred, vm, nil)
}
func (h hostLocal) ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount, input *api.ServerCreateInput) error {
hp := vm.HostPath
if hp == nil {
return httperrors.NewNotEmptyError("host_path is nil")
}
if hp.Path == "" {
return httperrors.NewNotEmptyError("path is required")
}
return nil
}

226
pkg/compute/guestdrivers/container.go
View File

@@ -1,226 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package guestdrivers
import (
"context"
"fmt"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/util/httputils"
"yunion.io/x/pkg/util/rbacscope"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
var (
containerUseKubectlError error = httperrors.NewUnsupportOperationError("Not supported, please use kubectl")
)
type SContainerDriver struct {
SVirtualizedGuestDriver
}
func init() {
driver := SContainerDriver{}
models.RegisterGuestDriver(&driver)
}
func (self *SContainerDriver) newUnsupportOperationError(option string) error {
return httperrors.NewUnsupportOperationError("Container not support %s", option)
}
func (self *SContainerDriver) GetHypervisor() string {
return api.HYPERVISOR_CONTAINER
}
func (self *SContainerDriver) GetProvider() string {
return api.CLOUD_PROVIDER_ONECLOUD
}
func (self *SContainerDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
return cloudprovider.SInstanceCapability{
Hypervisor: self.GetHypervisor(),
Provider: self.GetProvider(),
}
}
// for backward compatibility, deprecated driver
func (self *SContainerDriver) GetComputeQuotaKeys(scope rbacscope.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
keys := models.SComputeResourceKeys{}
keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)
keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE
keys.Provider = api.CLOUD_PROVIDER_ONECLOUD
keys.Brand = api.ONECLOUD_BRAND_ONECLOUD
keys.Hypervisor = api.HYPERVISOR_CONTAINER
return keys
}
func (self *SContainerDriver) GetDefaultSysDiskBackend() string {
return api.STORAGE_LOCAL
}
func (self *SContainerDriver) GetMinimalSysDiskSizeGb() int {
return options.Options.DefaultDiskSizeMB / 1024
}
func (self *SContainerDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
// do nothing, call next stage
task.ScheduleRun(nil)
return nil
}
func (self *SContainerDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, boot bool, task taskman.ITask) error {
// do nothing, call next stage
task.ScheduleRun(nil)
return nil
}
func (self *SContainerDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) error {
return httperrors.NewUnsupportOperationError("")
}
func (self *SContainerDriver) RequestStopOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask, syncStatus bool) error {
return containerUseKubectlError
}
func (self *SContainerDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
return containerUseKubectlError
}
func (self *SContainerDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
return containerUseKubectlError
}
func (self *SContainerDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, disk *models.SDisk, task taskman.ITask) error {
return containerUseKubectlError
}
func (self *SContainerDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) error {
taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
// always return running
status := jsonutils.NewDict()
status.Add(jsonutils.NewString("running"), "status")
return status, nil
})
return nil
}
func (self *SContainerDriver) CanKeepDetachDisk() bool {
return false
}
func (self *SContainerDriver) GetGuestVncInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost, input *cloudprovider.ServerVncInput) (*cloudprovider.ServerVncOutput, error) {
return nil, self.newUnsupportOperationError("VNC")
}
func (self *SContainerDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error {
//guest.SaveDeployInfo(ctx, task.GetUserCred(), data)
// do nothing here
return nil
}
func (self *SContainerDriver) RequestStopGuestForDelete(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
// do nothing, call next stage
task.ScheduleRun(nil)
return nil
}
func (self *SContainerDriver) RequestDetachDisksFromGuestForDelete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
// do nothing, call next stage
task.ScheduleRun(nil)
return nil
}
func (self *SContainerDriver) RequestUndeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
url := fmt.Sprintf("%s/servers/%s", host.ManagerUri, guest.Id)
header := self.getTaskRequestHeader(task)
_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "DELETE", url, header, nil, false)
return err
}
func (self *SContainerDriver) GetJsonDescAtHost(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost, params *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
desc := guest.GetJsonDescAtHypervisor(ctx, host)
return jsonutils.Marshal(desc), nil
}
func (self *SContainerDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
config, err := guest.GetDeployConfigOnHost(ctx, task.GetUserCred(), host, task.GetParams())
if err != nil {
log.Errorf("GetDeployConfigOnHost error: %v", err)
return err
}
config.Add(jsonutils.JSONTrue, "k8s_pod")
action, err := config.GetString("action")
if err != nil {
return err
}
url := fmt.Sprintf("%s/servers/%s/%s", host.ManagerUri, guest.Id, action)
header := self.getTaskRequestHeader(task)
_, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, config, false)
return err
}
func (self *SContainerDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error {
// clean disk records in DB
return guest.DeleteAllDisksInDB(ctx, userCred)
}
func (self *SContainerDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
// do nothing, call next stage
task.ScheduleRun(nil)
return nil
}
func (self *SContainerDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
return self.newUnsupportOperationError("create disk")
}
func (self *SContainerDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, instanceType string, vcpuCount, cpuSockets, vmemSize int64) error {
return self.newUnsupportOperationError("change config")
}
func (self *SContainerDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
// do nothing, call next stage
return self.newUnsupportOperationError("rebuild root")
}
func (self *SContainerDriver) GetRandomNetworkTypes() []string {
return []string{api.NETWORK_TYPE_CONTAINER, api.NETWORK_TYPE_GUEST}
}
func (self *SContainerDriver) StartGuestRestartTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isForce bool, parentTaskId string) error {
return fmt.Errorf("Not Implement")
}
func (self *SContainerDriver) IsSupportGuestClone() bool {
return false
}
func (self *SContainerDriver) IsSupportCdrom(guest *models.SGuest) (bool, error) {
return false, nil
}
func (self *SContainerDriver) IsSupportFloppy(guest *models.SGuest) (bool, error) {
return false, nil
}

413
pkg/compute/guestdrivers/pod.go Normal file
View File

@@ -0,0 +1,413 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package guestdrivers
import (
"context"
"fmt"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/httputils"
"yunion.io/x/pkg/util/rbacscope"
"yunion.io/x/pkg/util/sets"
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
var _ models.IPodDriver = new(SPodDriver)
type SPodDriver struct {
SKVMGuestDriver
}
func init() {
driver := SPodDriver{}
models.RegisterGuestDriver(&driver)
}
func (p *SPodDriver) newUnsupportOperationError(option string) error {
return httperrors.NewUnsupportOperationError("Container not support %s", option)
}
func (p *SPodDriver) GetHypervisor() string {
return api.HYPERVISOR_POD
}
func (p *SPodDriver) GetProvider() string {
return api.CLOUD_PROVIDER_ONECLOUD
}
func (p *SPodDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) {
if input.Pod == nil {
return nil, httperrors.NewNotEmptyError("pod data is empty")
}
if len(input.Pod.Containers) == 0 {
return nil, httperrors.NewNotEmptyError("containers data is empty")
}
// validate port mappings
if err := p.validatePortMappings(input.Pod); err != nil {
return nil, errors.Wrap(err, "validate port mappings")
}
sameName := ""
for idx, ctr := range input.Pod.Containers {
if err := p.validateContainerData(ctx, userCred, idx, input.Name, ctr, input); err != nil {
return nil, errors.Wrapf(err, "data of %d container", idx)
}
if ctr.Name == sameName {
return nil, httperrors.NewDuplicateNameError("same name %s of containers", ctr.Name)
}
sameName = ctr.Name
}
// always set auto_start to true
input.AutoStart = true
return input, nil
}
func (p *SPodDriver) validatePortMappings(input *api.PodCreateInput) error {
for idx, pm := range input.PortMappings {
// TODO: 判断 host port 是否重复
if err := p.validatePortMapping(pm); err != nil {
return errors.Wrapf(err, "validate portmapping %d", idx)
}
}
return nil
}
func (p *SPodDriver) validateContainerData(ctx context.Context, userCred mcclient.TokenCredential, idx int, defaultNamePrefix string, ctr *api.PodContainerCreateInput, input *api.ServerCreateInput) error {
if ctr.Name == "" {
ctr.Name = fmt.Sprintf("%s-%d", defaultNamePrefix, idx)
}
if err := models.GetContainerManager().ValidateSpec(ctx, userCred, &ctr.ContainerSpec, nil); err != nil {
return errors.Wrap(err, "validate container spec")
}
if err := p.validateContainerVolumeMounts(ctx, userCred, ctr, input); err != nil {
return errors.Wrap(err, "validate container volumes")
}
return nil
}
func (p *SPodDriver) validateContainerVolumeMounts(ctx context.Context, userCred mcclient.TokenCredential, ctr *api.PodContainerCreateInput, input *api.ServerCreateInput) error {
for idx, vm := range ctr.VolumeMounts {
if err := p.validateContainerVolumeMount(ctx, userCred, vm, input); err != nil {
return errors.Wrapf(err, "validate volume mount %d", idx)
}
}
return nil
}
func (p *SPodDriver) validateContainerVolumeMount(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount, input *api.ServerCreateInput) error {
if vm.Type == "" {
return httperrors.NewNotEmptyError("type is required")
}
if vm.MountPath == "" {
return httperrors.NewNotEmptyError("mount_path is required")
}
drv, err := models.GetContainerVolumeMountDriverWithError(vm.Type)
if err != nil {
return errors.Wrapf(err, "get container volume mount driver %s", vm.Type)
}
if err := drv.ValidatePodCreateData(ctx, userCred, vm, input); err != nil {
return errors.Wrapf(err, "validate %s create data", vm.Type)
}
return nil
}
func (p *SPodDriver) validatePortRange(port int32) error {
if port <= 0 || port > 65535 {
return httperrors.NewInputParameterError("port number %d isn't within 1 to 65535", port)
}
return nil
}
func (p *SPodDriver) validatePortMapping(pm *api.PodPortMapping) error {
if err := p.validatePortRange(pm.HostPort); err != nil {
return errors.Wrap(err, "validate host_port")
}
if err := p.validatePortRange(pm.ContainerPort); err != nil {
return errors.Wrap(err, "validate container_port")
}
if pm.Protocol == "" {
pm.Protocol = api.PodPortMappingProtocolTCP
}
if !sets.NewString(api.PodPortMappingProtocolSCTP, api.PodPortMappingProtocolUDP, api.PodPortMappingProtocolTCP).Has(string(pm.Protocol)) {
return httperrors.NewInputParameterError("unsupported protocol %s", pm.Protocol)
}
return nil
}
func (p *SPodDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
return cloudprovider.SInstanceCapability{
Hypervisor: p.GetHypervisor(),
Provider: p.GetProvider(),
}
}
// for backward compatibility, deprecated driver
func (p *SPodDriver) GetComputeQuotaKeys(scope rbacscope.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
keys := models.SComputeResourceKeys{}
keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)
keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE
keys.Provider = api.CLOUD_PROVIDER_ONECLOUD
keys.Brand = api.ONECLOUD_BRAND_ONECLOUD
keys.Hypervisor = api.HYPERVISOR_POD
return keys
}
func (p *SPodDriver) GetDefaultSysDiskBackend() string {
return api.STORAGE_LOCAL
}
func (p *SPodDriver) GetMinimalSysDiskSizeGb() int {
return options.Options.DefaultDiskSizeMB / 1024
}
func (p *SPodDriver) StartGuestCreateTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, pendingUsage quotas.IQuota, parentTaskId string) error {
task, err := taskman.TaskManager.NewTask(ctx, "PodCreateTask", guest, userCred, data, parentTaskId, "", pendingUsage)
if err != nil {
return errors.Wrap(err, "New PodCreateTask")
}
return task.ScheduleRun(nil)
}
func (p *SPodDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, boot bool, task taskman.ITask) error {
// do nothing, call next stage
return task.ScheduleRun(nil)
}
func (p *SPodDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) error {
header := p.getTaskRequestHeader(task)
config := jsonutils.NewDict()
desc, err := guest.GetDriver().GetJsonDescAtHost(ctx, userCred, guest, host, nil)
if err != nil {
return errors.Wrapf(err, "GetJsonDescAtHost")
}
config.Add(desc, "desc")
params := task.GetParams()
if params.Length() > 0 {
config.Add(params, "params")
}
url := fmt.Sprintf("%s/servers/%s/start", host.ManagerUri, guest.Id)
_, body, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, config, false)
if err != nil {
return err
}
resp := new(api.PodStartResponse)
body.Unmarshal(resp)
if resp.IsRunning {
taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
return body, nil
})
}
return nil
}
func (p *SPodDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
return p.newUnsupportOperationError("suspend")
}
func (p *SPodDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
return p.newUnsupportOperationError("soft reset")
}
func (p *SPodDriver) GetGuestVncInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost, input *cloudprovider.ServerVncInput) (*cloudprovider.ServerVncOutput, error) {
return nil, p.newUnsupportOperationError("VNC")
}
func (p *SPodDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error {
//guest.SaveDeployInfo(ctx, task.GetUserCred(), data)
// do nothing here
return nil
}
func (p *SPodDriver) RequestUndeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
task, err := taskman.TaskManager.NewTask(ctx, "PodDeleteTask", guest, task.GetUserCred(), nil, task.GetTaskId(), "", nil)
if err != nil {
return errors.Wrap(err, "New PodDeleteTask")
}
return task.ScheduleRun(nil)
}
func (p *SPodDriver) RequestUndeployPod(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
url := fmt.Sprintf("%s/servers/%s", host.ManagerUri, guest.Id)
header := p.getTaskRequestHeader(task)
_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "DELETE", url, header, nil, false)
return err
}
func (p *SPodDriver) GetJsonDescAtHost(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost, params *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
desc := guest.GetJsonDescAtHypervisor(ctx, host)
ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId())
if err != nil {
return nil, errors.Wrap(err, "GetContainersByPod")
}
ctrDescs := make([]*api.ContainerDesc, len(ctrs))
for idx, ctr := range ctrs {
ctrDescs[idx] = ctr.GetJsonDescAtHost()
}
desc.Containers = ctrDescs
return jsonutils.Marshal(desc), nil
}
func (p *SPodDriver) createContainersOnPod(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest) error {
input, err := guest.GetCreateParams(ctx, userCred)
if err != nil {
return errors.Wrap(err, "GetCreateParams")
}
ctrs := make([]*models.SContainer, len(input.Pod.Containers))
for idx, ctr := range input.Pod.Containers {
if obj, err := models.GetContainerManager().CreateOnPod(ctx, userCred, guest.GetOwnerId(), guest, ctr); err != nil {
return errors.Wrapf(err, "create container on pod: %s", guest.GetName())
} else {
ctrs[idx] = obj
}
}
return nil
}
func (p *SPodDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
deployAction, err := task.GetParams().GetString("deploy_action")
if err != nil {
return errors.Wrapf(err, "get deploy_action from task params: %s", task.GetParams())
}
if deployAction == "create" {
if err := p.createContainersOnPod(ctx, task.GetUserCred(), guest); err != nil {
return errors.Wrap(err, "create containers on pod")
}
}
config, err := guest.GetDeployConfigOnHost(ctx, task.GetUserCred(), host, task.GetParams())
if err != nil {
log.Errorf("GetDeployConfigOnHost error: %v", err)
return err
}
action, err := config.GetString("action")
if err != nil {
return err
}
url := fmt.Sprintf("%s/servers/%s/%s", host.ManagerUri, guest.Id, action)
header := p.getTaskRequestHeader(task)
_, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, config, false)
return err
}
func (p *SPodDriver) performContainerAction(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask, action string, data jsonutils.JSONObject) error {
pod := task.GetPod()
ctr := task.GetContainer()
host, _ := pod.GetHost()
url := fmt.Sprintf("%s/pods/%s/containers/%s/%s", host.ManagerUri, pod.GetId(), ctr.GetId(), action)
header := p.getTaskRequestHeader(task)
_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, data, false)
return err
}
func (p *SPodDriver) getContainerCreateInput(ctx context.Context, userCred mcclient.TokenCredential, ctr *models.SContainer) (*hostapi.ContainerCreateInput, error) {
spec, err := ctr.ToHostContainerSpec(ctx, userCred)
if err != nil {
return nil, errors.Wrap(err, "ToHostContainerSpec")
}
input := &hostapi.ContainerCreateInput{
Name: ctr.GetName(),
GuestId: ctr.GuestId,
Spec: spec,
}
return input, nil
}
func (p *SPodDriver) RequestCreateContainer(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error {
ctr := task.GetContainer()
input, err := p.getContainerCreateInput(ctx, userCred, ctr)
if err != nil {
return errors.Wrap(err, "getContainerCreateInput")
}
return p.performContainerAction(ctx, userCred, task, "create", jsonutils.Marshal(input))
}
func (p *SPodDriver) RequestStartContainer(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error {
ctr := task.GetContainer()
input, err := p.getContainerCreateInput(ctx, userCred, ctr)
if err != nil {
return errors.Wrap(err, "getContainerCreateInput")
}
return p.performContainerAction(ctx, userCred, task, "start", jsonutils.Marshal(input))
}
func (p *SPodDriver) RequestStopContainer(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error {
return p.performContainerAction(ctx, userCred, task, "stop", task.GetParams())
}
func (p *SPodDriver) RequestDeleteContainer(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error {
return p.performContainerAction(ctx, userCred, task, "delete", nil)
}
func (p *SPodDriver) RequestSyncContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error {
return p.performContainerAction(ctx, userCred, task, "sync-status", nil)
}
func (p *SPodDriver) RequestPullContainerImage(ctx context.Context, userCred mcclient.TokenCredential, task models.IContainerTask) error {
return p.performContainerAction(ctx, userCred, task, "pull-image", task.GetParams())
}
func (p *SPodDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error {
// clean disk records in DB
return guest.DeleteAllDisksInDB(ctx, userCred)
}
func (p *SPodDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
// do nothing, call next stage
task.ScheduleRun(nil)
return nil
}
func (p *SPodDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, instanceType string, vcpuCount, cpuSockets, vmemSize int64) error {
return p.newUnsupportOperationError("change config")
}
func (p *SPodDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
// do nothing, call next stage
return p.newUnsupportOperationError("rebuild root")
}
func (p *SPodDriver) GetRandomNetworkTypes() []string {
return []string{api.NETWORK_TYPE_CONTAINER, api.NETWORK_TYPE_GUEST}
}
func (p *SPodDriver) StartGuestRestartTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isForce bool, parentTaskId string) error {
return fmt.Errorf("Not Implement")
}
func (p *SPodDriver) IsSupportGuestClone() bool {
return false
}
func (p *SPodDriver) IsSupportCdrom(guest *models.SGuest) (bool, error) {
return false, nil
}
func (p *SPodDriver) IsSupportFloppy(guest *models.SGuest) (bool, error) {
return false, nil
}

25
pkg/compute/hostdrivers/container.go Normal file
View File

@@ -0,0 +1,25 @@
package hostdrivers
import (
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/compute/models"
)
func init() {
driver := &SContainerHostDriver{
SKVMHostDriver: &SKVMHostDriver{},
}
models.RegisterHostDriver(driver)
}
type SContainerHostDriver struct {
*SKVMHostDriver
}
func (d *SContainerHostDriver) GetHostType() string {
return api.HOST_TYPE_CONTAINER
}
func (d *SContainerHostDriver) GetHypervisor() string {
return api.HYPERVISOR_POD
}

69
pkg/compute/models/container_drivers.go Normal file
View File

@@ -0,0 +1,69 @@
package models
import (
"context"
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
var (
containerVolumeDrivers = make(map[apis.ContainerVolumeMountType]IContainerVolumeMountDriver)
containerDeviceDrivers = make(map[apis.ContainerDeviceType]IContainerDeviceDriver)
)
func RegisterContainerVolumeMountDriver(drv IContainerVolumeMountDriver) {
containerVolumeDrivers[drv.GetType()] = drv
}
func GetContainerVolumeMountDriver(typ apis.ContainerVolumeMountType) IContainerVolumeMountDriver {
drv, err := GetContainerVolumeMountDriverWithError(typ)
if err != nil {
panic(err.Error())
}
return drv
}
func GetContainerVolumeMountDriverWithError(typ apis.ContainerVolumeMountType) (IContainerVolumeMountDriver, error) {
drv, ok := containerVolumeDrivers[typ]
if !ok {
return nil, httperrors.NewNotFoundError("not found driver by type %q", typ)
}
return drv, nil
}
type IContainerVolumeMountDriver interface {
GetType() apis.ContainerVolumeMountType
ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount, input *api.ServerCreateInput) error
ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error)
}
func RegisterContainerDeviceDriver(drv IContainerDeviceDriver) {
containerDeviceDrivers[drv.GetType()] = drv
}
func GetContainerDeviceDriver(typ apis.ContainerDeviceType) IContainerDeviceDriver {
drv, err := GetContainerDeviceDriverWithError(typ)
if err != nil {
panic(err.Error())
}
return drv
}
func GetContainerDeviceDriverWithError(typ apis.ContainerDeviceType) (IContainerDeviceDriver, error) {
drv, ok := containerDeviceDrivers[typ]
if !ok {
return nil, httperrors.NewNotFoundError("not found driver by type %q", typ)
}
return drv, nil
}
type IContainerDeviceDriver interface {
GetType() apis.ContainerDeviceType
ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, dev *api.ContainerDevice, input *api.ServerCreateInput) error
ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error)
ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error)
}

382
pkg/compute/models/containers.go Normal file
View File

@@ -0,0 +1,382 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/sets"
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
var containerManager *SContainerManager
func GetContainerManager() *SContainerManager {
if containerManager == nil {
containerManager = &SContainerManager{
SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
SContainer{},
"containers_tbl",
"container",
"containers"),
}
containerManager.SetVirtualObject(containerManager)
}
return containerManager
}
func init() {
GetContainerManager()
}
type SContainerManager struct {
db.SVirtualResourceBaseManager
}
type SContainer struct {
db.SVirtualResourceBase
// GuestId is also the pod id
GuestId string `width:"36" charset:"ascii" create:"required" list:"user" index:"true"`
// Spec stores all container running options
Spec *api.ContainerSpec `length:"long" create:"required" list:"user"`
}
func (m *SContainerManager) CreateOnPod(
ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
pod *SGuest, data *api.PodContainerCreateInput) (*SContainer, error) {
input := &api.ContainerCreateInput{
GuestId: pod.GetId(),
Spec: data.ContainerSpec,
SkipTask: true,
}
input.Name = data.Name
obj, err := db.DoCreate(m, ctx, userCred, nil, jsonutils.Marshal(input), ownerId)
if err != nil {
return nil, errors.Wrap(err, "create container")
}
return obj.(*SContainer), nil
}
func (m *SContainerManager) FetchUniqValues(ctx context.Context, data jsonutils.JSONObject) jsonutils.JSONObject {
guestId, _ := data.GetString("guest_id")
return jsonutils.Marshal(map[string]string{"guest_id": guestId})
}
func (m *SContainerManager) FilterByUniqValues(q *sqlchemy.SQuery, values jsonutils.JSONObject) *sqlchemy.SQuery {
guestId, _ := values.GetString("guest_id")
if len(guestId) > 0 {
q = q.Equals("guest_id", guestId)
}
return q
}
func (m *SContainerManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query api.ContainerListInput) (*sqlchemy.SQuery, error) {
q, err := m.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.VirtualResourceListInput)
if err != nil {
return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter")
}
return q, nil
}
func (m *SContainerManager) GetContainersByPod(guestId string) ([]SContainer, error) {
q := m.Query().Equals("guest_id", guestId)
ctrs := make([]SContainer, 0)
if err := db.FetchModelObjects(m, q, &ctrs); err != nil {
return nil, errors.Wrap(err, "db.FetchModelObjects")
}
return ctrs, nil
}
func (m *SContainerManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, _ jsonutils.JSONObject, input *api.ContainerCreateInput) (*api.ContainerCreateInput, error) {
if input.GuestId == "" {
return nil, httperrors.NewNotEmptyError("guest_id is required")
}
obj, err := GuestManager.FetchByIdOrName(ctx, userCred, input.GuestId)
if err != nil {
return nil, errors.Wrapf(err, "fetch guest by %s", input.GuestId)
}
pod := obj.(*SGuest)
input.GuestId = pod.GetId()
if err := m.ValidateSpec(ctx, userCred, &input.Spec, pod); err != nil {
return nil, errors.Wrap(err, "validate spec")
}
return input, nil
}
func (m *SContainerManager) ValidateSpec(ctx context.Context, userCred mcclient.TokenCredential, spec *api.ContainerSpec, pod *SGuest) error {
if spec.ImagePullPolicy == "" {
spec.ImagePullPolicy = apis.ImagePullPolicyIfNotPresent
}
if !sets.NewString(apis.ImagePullPolicyAlways, apis.ImagePullPolicyIfNotPresent).Has(string(spec.ImagePullPolicy)) {
return httperrors.NewInputParameterError("invalid image_pull_policy %s", spec.ImagePullPolicy)
}
if pod != nil {
if err := m.ValidateSpecVolumeMounts(ctx, userCred, pod, spec); err != nil {
return errors.Wrap(err, "ValidateSpecVolumeMounts")
}
for idx, dev := range spec.Devices {
newDev, err := m.ValidateSpecDevice(ctx, userCred, pod, dev)
if err != nil {
return errors.Wrapf(err, "validate device %s", jsonutils.Marshal(dev))
}
spec.Devices[idx] = newDev
}
}
return nil
}
func (m *SContainerManager) ValidateSpecDevice(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) {
drv, err := GetContainerDeviceDriverWithError(dev.Type)
if err != nil {
return nil, httperrors.NewInputParameterError("get device drvice: %v", err)
}
return drv.ValidateCreateData(ctx, userCred, pod, dev)
}
func (m *SContainerManager) ValidateSpecVolumeMounts(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, spec *api.ContainerSpec) error {
relation, err := m.GetVolumeMountRelations(ctx, userCred, pod, spec)
if err != nil {
return errors.Wrap(err, "GetVolumeMountRelations")
}
for idx, vm := range spec.VolumeMounts {
newVm, err := m.ValidateSpecVolumeMount(ctx, userCred, pod, vm)
if err != nil {
return errors.Wrapf(err, "validate volume mount %s", jsonutils.Marshal(vm))
}
spec.VolumeMounts[idx] = newVm
}
if _, err := m.ConvertVolumeMountRelationToSpec(relation); err != nil {
return errors.Wrap(err, "ConvertVolumeMountRelationToSpec")
}
return nil
}
func (m *SContainerManager) ValidateSpecVolumeMount(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) {
if vm.Type == "" {
return nil, httperrors.NewNotEmptyError("type is required")
}
if vm.MountPath == "" {
return nil, httperrors.NewNotEmptyError("mount_path is required")
}
drv, err := GetContainerVolumeMountDriverWithError(vm.Type)
if err != nil {
return nil, errors.Wrapf(err, "get container volume mount driver %s", vm.Type)
}
vm, err = drv.ValidateCreateData(ctx, userCred, pod, vm)
if err != nil {
return nil, errors.Wrapf(err, "validate %s create data", vm.Type)
}
return vm, nil
}
/*func (m *SContainerManager) GetContainerIndex(guestId string) (int, error) {
cnt, err := m.Query("guest_id").Equals("guest_id", guestId).CountWithError()
if err != nil {
return -1, errors.Wrapf(err, "get container numbers of pod %s", guestId)
}
return cnt, nil
}
func (c *SContainer) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
input := new(api.ContainerCreateInput)
if err := data.Unmarshal(input); err != nil {
return errors.Wrap(err, "unmarshal to ContainerCreateInput")
}
if input.Spec.ImagePullPolicy == "" {
c.Spec.ImagePullPolicy = apis.ImagePullPolicyIfNotPresent
}
return nil
}*/
func (c *SContainer) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
c.SVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
if !jsonutils.QueryBoolean(data, "skip_task", false) {
if err := c.StartCreateTask(ctx, userCred, ""); err != nil {
log.Errorf("StartCreateTask error: %v", err)
}
}
}
func (c *SContainer) StartCreateTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
task, err := taskman.TaskManager.NewTask(ctx, "ContainerCreateTask", c, userCred, nil, parentTaskId, "", nil)
if err != nil {
return errors.Wrap(err, "NewTask")
}
return task.ScheduleRun(nil)
}
func (c *SContainer) GetPod() *SGuest {
return GuestManager.FetchGuestById(c.GuestId)
}
func (c *SContainer) GetVolumeMounts() []*apis.ContainerVolumeMount {
return c.Spec.VolumeMounts
}
type ContainerVolumeMountRelation struct {
VolumeMount *apis.ContainerVolumeMount
pod *SGuest
}
func (vm *ContainerVolumeMountRelation) ToHostMount() (*apis.ContainerVolumeMount, error) {
return vm.VolumeMount, nil
}
func (m *SContainerManager) GetVolumeMountRelations(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, spec *api.ContainerSpec) ([]*ContainerVolumeMountRelation, error) {
relation := make([]*ContainerVolumeMountRelation, len(spec.VolumeMounts))
for idx, vm := range spec.VolumeMounts {
tmpVm := vm
relation[idx] = &ContainerVolumeMountRelation{
VolumeMount: tmpVm,
pod: pod,
}
}
return relation, nil
}
func (c *SContainer) GetVolumeMountRelations(ctx context.Context, userCred mcclient.TokenCredential) ([]*ContainerVolumeMountRelation, error) {
return GetContainerManager().GetVolumeMountRelations(ctx, userCred, c.GetPod(), c.Spec)
}
func (c *SContainer) PerformStart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
if !sets.NewString(api.CONTAINER_STATUS_EXITED, api.CONTAINER_STATUS_START_FAILED).Has(c.Status) {
return nil, httperrors.NewInvalidStatusError("Can't start container in status %s", c.Status)
}
return nil, c.StartStartTask(ctx, userCred, "")
}
func (c *SContainer) StartStartTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_STARTING, "")
task, err := taskman.TaskManager.NewTask(ctx, "ContainerStartTask", c, userCred, nil, parentTaskId, "", nil)
if err != nil {
return errors.Wrap(err, "NewTask")
}
return task.ScheduleRun(nil)
}
func (c *SContainer) PerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *api.ContainerStopInput) (jsonutils.JSONObject, error) {
if !sets.NewString(api.CONTAINER_STATUS_RUNNING, api.CONTAINER_STATUS_STOP_FAILED).Has(c.Status) {
return nil, httperrors.NewInvalidStatusError("Can't stop container in status %s", c.Status)
}
return nil, c.StartStopTask(ctx, userCred, data, "")
}
func (c *SContainer) StartStopTask(ctx context.Context, userCred mcclient.TokenCredential, data *api.ContainerStopInput, parentTaskId string) error {
c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_STOPPING, "")
task, err := taskman.TaskManager.NewTask(ctx, "ContainerStopTask", c, userCred, jsonutils.Marshal(data).(*jsonutils.JSONDict), parentTaskId, "", nil)
if err != nil {
return errors.Wrap(err, "NewTask")
}
return task.ScheduleRun(nil)
}
func (c *SContainer) StartSyncStatusTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_SYNC_STATUS, "")
task, err := taskman.TaskManager.NewTask(ctx, "ContainerSyncStatusTask", c, userCred, nil, parentTaskId, "", nil)
if err != nil {
return errors.Wrap(err, "NewTask")
}
return task.ScheduleRun(nil)
}
func (c *SContainer) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query, data jsonutils.JSONObject) error {
return c.StartDeleteTask(ctx, userCred, "")
}
func (c *SContainer) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_DELETING, "")
task, err := taskman.TaskManager.NewTask(ctx, "ContainerDeleteTask", c, userCred, nil, parentTaskId, "", nil)
if err != nil {
return errors.Wrap(err, "NewTask")
}
return task.ScheduleRun(nil)
}
func (c *SContainer) StartPullImageTask(ctx context.Context, userCred mcclient.TokenCredential, input *hostapi.ContainerPullImageInput, parentTaskId string) error {
c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_PULLING_IMAGE, "")
task, err := taskman.TaskManager.NewTask(ctx, "ContainerPullImageTask", c, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil)
if err != nil {
return errors.Wrap(err, "NewTask")
}
return task.ScheduleRun(nil)
}
func (c *SContainer) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
return c.SVirtualResourceBase.Delete(ctx, userCred)
}
func (m *SContainerManager) ConvertVolumeMountRelationToSpec(relation []*ContainerVolumeMountRelation) ([]*apis.ContainerVolumeMount, error) {
mounts := make([]*apis.ContainerVolumeMount, 0)
for _, r := range relation {
mount, err := r.ToHostMount()
if err != nil {
return nil, errors.Wrapf(err, "ToMountOrDevice: %#v", r)
}
if mount != nil {
mounts = append(mounts, mount)
}
}
return mounts, nil
}
func (c *SContainer) ToHostContainerSpec(ctx context.Context, userCred mcclient.TokenCredential) (*hostapi.ContainerSpec, error) {
vmRelation, err := c.GetVolumeMountRelations(ctx, userCred)
if err != nil {
return nil, errors.Wrap(err, "GetVolumeMountRelations")
}
mounts, err := GetContainerManager().ConvertVolumeMountRelationToSpec(vmRelation)
if err != nil {
return nil, errors.Wrap(err, "ConvertVolumeRelationToSpec")
}
spec := c.Spec.ContainerSpec
spec.VolumeMounts = mounts
hSpec := &hostapi.ContainerSpec{
ContainerSpec: spec,
}
ctrDevs := make([]*hostapi.ContainerDevice, 0)
for _, dev := range c.Spec.Devices {
ctrDev, err := GetContainerDeviceDriver(dev.Type).ToHostDevice(dev)
if err != nil {
return nil, errors.Wrapf(err, "ToHostDevice %s", jsonutils.Marshal(dev))
}
ctrDevs = append(ctrDevs, ctrDev)
}
hSpec.Devices = ctrDevs
return hSpec, nil
}
func (c *SContainer) GetJsonDescAtHost() *api.ContainerDesc {
return &api.ContainerDesc{
Id: c.GetId(),
Name: c.GetName(),
Spec: c.Spec,
}
}

17
pkg/compute/models/guests.go
View File

@@ -1497,7 +1497,7 @@ func (manager *SGuestManager) validateCreateData(
// var rootStorageType string
var osProf osprofile.SOSProfile
hypervisor = input.Hypervisor
if hypervisor != api.HYPERVISOR_CONTAINER {
if hypervisor != api.HYPERVISOR_POD {
if len(input.Disks) == 0 && input.Cdrom == "" {
return nil, httperrors.NewInputParameterError("No bootable disk information provided")
}
@@ -1631,7 +1631,7 @@ func (manager *SGuestManager) validateCreateData(
return nil, err
}
optionSystemHypervisor := []string{api.HYPERVISOR_KVM, api.HYPERVISOR_ESXI}
optionSystemHypervisor := []string{api.HYPERVISOR_KVM, api.HYPERVISOR_ESXI, api.HYPERVISOR_POD}
if !utils.IsInStringArray(input.Hypervisor, optionSystemHypervisor) && len(input.Disks[0].ImageId) == 0 && len(input.Disks[0].SnapshotId) == 0 && input.Cdrom == "" {
return nil, httperrors.NewBadRequestError("Miss operating system???")
@@ -1645,7 +1645,7 @@ func (manager *SGuestManager) validateCreateData(
}
hypervisor = input.Hypervisor
if hypervisor != api.HYPERVISOR_CONTAINER {
if hypervisor != api.HYPERVISOR_POD {
// support sku here
var sku *SServerSku
skuName := input.InstanceType
@@ -4760,9 +4760,11 @@ func (self *SGuest) DeleteAllInstanceSnapshotInDB(ctx context.Context, userCred
func (self *SGuest) isNeedDoResetPasswd() bool {
guestdisks, _ := self.GetGuestDisks()
disk := guestdisks[0].GetDisk()
if len(disk.SnapshotId) > 0 {
return false
if len(guestdisks) > 0 {
disk := guestdisks[0].GetDisk()
if len(disk.SnapshotId) > 0 {
return false
}
}
return true
}
@@ -5000,7 +5002,8 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *a
IsDaemon: self.IsDaemon.Bool(),
LightMode: self.RescueMode,
LightMode: self.RescueMode,
Hypervisor: self.GetHypervisor(),
}
if len(self.BackupHostId) > 0 {

2
pkg/compute/models/helper.go
View File

@@ -62,7 +62,7 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred
}
// base validate_create_data
if (input.PreferHost != "") && hypervisor != api.HYPERVISOR_CONTAINER {
if (input.PreferHost != "") && hypervisor != api.HYPERVISOR_POD {
bmName := input.PreferHost
bmObj, err := HostManager.FetchByIdOrName(ctx, nil, bmName)

6
pkg/compute/models/hosts.go
View File

@@ -1651,7 +1651,7 @@ func (hh *SHost) GetGuestCount() (int, error) {
func (hh *SHost) GetContainerCount(status []string) (int, error) {
q := hh.GetGuestsQuery()
q = q.Filter(sqlchemy.Equals(q.Field("hypervisor"), api.HYPERVISOR_CONTAINER))
q = q.Filter(sqlchemy.Equals(q.Field("hypervisor"), api.HYPERVISOR_POD))
if len(status) > 0 {
q = q.In("status", status)
}
@@ -3214,7 +3214,7 @@ func (manager *SHostManager) FetchGuestCnt(hostIds []string) map[string]*sGuestC
return ret
}
guests := []SGuest{}
err := GuestManager.RawQuery().IsFalse("deleted").In("host_id", hostIds).NotEquals("hypervisor", api.HYPERVISOR_CONTAINER).All(&guests)
err := GuestManager.RawQuery().IsFalse("deleted").In("host_id", hostIds).NotEquals("hypervisor", api.HYPERVISOR_POD).All(&guests)
if err != nil {
log.Errorf("query host %s guests error: %v", hostIds, err)
}
@@ -3241,7 +3241,7 @@ func (manager *SHostManager) FetchGuestCnt(hostIds []string) map[string]*sGuestC
}
}
GuestManager.RawQuery().IsFalse("deleted").In("backup_host_id", hostIds).NotEquals("hypervisor", api.HYPERVISOR_CONTAINER).All(&guests)
GuestManager.RawQuery().IsFalse("deleted").In("backup_host_id", hostIds).NotEquals("hypervisor", api.HYPERVISOR_POD).All(&guests)
for _, guest := range guests {
_, ok := ret[guest.BackupHostId]
if !ok {

7
pkg/compute/models/isolated_devices.go
View File

@@ -89,7 +89,7 @@ type SIsolatedDevice struct {
// # PCI / GPU-HPC / GPU-VGA / USB / NIC
// 设备类型
DevType string `width:"16" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required" update:"domain"`
DevType string `width:"36" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required" update:"domain"`
// # Specific device name read from lspci command, e.g. `Tesla K40m` ...
Model string `width:"512" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required" update:"domain"`
@@ -110,7 +110,8 @@ type SIsolatedDevice struct {
DiskIndex int8 `nullable:"true" default:"-1" list:"user" update:"user"`
// # pci address of `Bus:Device.Function` format, or usb bus address of `bus.addr`
Addr string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
Addr string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
DevicePath string `width:"128" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"optional"`
// Is vgpu physical funcion, That means it cannot be attached to guest
// VGPUPhysicalFunction bool `nullable:"true" default:"false" list:"domain" create:"domain_optional"`
@@ -324,7 +325,7 @@ func (manager *SIsolatedDeviceManager) ListItemFilter(
}
if !query.ShowBaremetalIsolatedDevices {
sq := HostManager.Query("id").Equals("host_type", api.HOST_TYPE_HYPERVISOR).SubQuery()
sq := HostManager.Query("id").In("host_type", []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_CONTAINER}).SubQuery()
q = q.In("host_id", sq)
}

26
pkg/compute/models/pod_driver.go Normal file
View File

@@ -0,0 +1,26 @@
package models
import (
"context"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/mcclient"
)
type IContainerTask interface {
taskman.ITask
GetContainer() *SContainer
GetPod() *SGuest
}
type IPodDriver interface {
IGuestDriver
RequestCreateContainer(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
RequestStartContainer(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
RequestStopContainer(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
RequestDeleteContainer(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
RequestSyncContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
RequestPullContainerImage(ctx context.Context, userCred mcclient.TokenCredential, task IContainerTask) error
}

2
pkg/compute/models/secgroups.go
View File

@@ -301,7 +301,7 @@ func (self *SSecurityGroup) GetGuestsQuery() *sqlchemy.SQuery {
sqlchemy.Equals(guests.Field("admin_secgrp_id"), self.Id),
sqlchemy.In(guests.Field("id"), GuestsecgroupManager.Query("guest_id").Equals("secgroup_id", self.Id).SubQuery()),
),
).Filter(sqlchemy.NotIn(guests.Field("hypervisor"), []string{api.HYPERVISOR_CONTAINER, api.HYPERVISOR_BAREMETAL, api.HYPERVISOR_ESXI}))
).Filter(sqlchemy.NotIn(guests.Field("hypervisor"), []string{api.HYPERVISOR_POD, api.HYPERVISOR_BAREMETAL, api.HYPERVISOR_ESXI}))
}
func (self *SSecurityGroup) GetGuestsCount() (int, error) {

1
pkg/compute/service/handlers.go
View File

@@ -124,6 +124,7 @@ func InitHandlers(app *appsrv.Application) {
models.HostManager,
models.SchedtagManager,
models.GuestManager,
models.GetContainerManager(),
models.GroupManager,
models.DiskManager,
models.NetworkManager,

2
pkg/compute/service/service.go
View File

@@ -39,6 +39,8 @@ import (
"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
_ "yunion.io/x/onecloud/pkg/compute/container_drivers/device"
_ "yunion.io/x/onecloud/pkg/compute/container_drivers/volume_mount"
_ "yunion.io/x/onecloud/pkg/compute/guestdrivers"
_ "yunion.io/x/onecloud/pkg/compute/hostdrivers"
"yunion.io/x/onecloud/pkg/compute/models"

105
pkg/compute/tasks/container_create_task.go Normal file
View File

@@ -0,0 +1,105 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
)
type ContainerBaseTask struct {
taskman.STask
}
func (t *ContainerBaseTask) GetContainer() *models.SContainer {
return t.GetObject().(*models.SContainer)
}
func (t *ContainerBaseTask) GetPod() *models.SGuest {
return t.GetContainer().GetPod()
}
func (t *ContainerBaseTask) GetPodDriver() models.IPodDriver {
return t.GetPod().GetDriver().(models.IPodDriver)
}
type ContainerCreateTask struct {
ContainerBaseTask
}
func init() {
taskman.RegisterTask(ContainerCreateTask{})
}
func (t *ContainerCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
t.startPullImage(ctx, obj.(*models.SContainer))
}
func (t *ContainerCreateTask) startPullImage(ctx context.Context, container *models.SContainer) {
t.SetStage("OnImagePulled", nil)
input := &hostapi.ContainerPullImageInput{
Image: container.Spec.Image,
PullPolicy: container.Spec.ImagePullPolicy,
}
if err := container.StartPullImageTask(ctx, t.GetUserCred(), input, t.GetTaskId()); err != nil {
t.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
return
}
}
func (t *ContainerCreateTask) OnImagePulled(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
t.requestCreate(ctx, container)
}
func (t *ContainerCreateTask) OnImagePulledFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
t.SetStageFailed(ctx, reason)
}
func (t *ContainerCreateTask) requestCreate(ctx context.Context, container *models.SContainer) {
t.SetStage("OnCreated", nil)
container.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_CREATING, "")
if err := t.GetPodDriver().RequestCreateContainer(ctx, t.GetUserCred(), t); err != nil {
t.OnCreatedFailed(ctx, container, jsonutils.NewString(err.Error()))
return
}
}
func (t *ContainerCreateTask) OnCreated(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
t.SetStage("OnStarted", nil)
if err := container.StartStartTask(ctx, t.GetUserCred(), t.GetTaskId()); err != nil {
t.OnCreatedFailed(ctx, container, jsonutils.NewString(errors.Wrap(err, "StartStartTask").Error()))
}
}
func (t *ContainerCreateTask) OnCreatedFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
container.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_CREATE_FAILED, reason.String())
t.SetStageFailed(ctx, reason)
}
func (t *ContainerCreateTask) OnStarted(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
t.SetStageComplete(ctx, nil)
}
func (t *ContainerCreateTask) OnStartedFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
t.SetStageFailed(ctx, reason)
}

60
pkg/compute/tasks/container_delete_task.go Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
)
type ContainerDeleteTask struct {
ContainerBaseTask
}
func init() {
taskman.RegisterTask(ContainerDeleteTask{})
}
func (t *ContainerDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
t.requestDelete(ctx, obj.(*models.SContainer))
}
func (t *ContainerDeleteTask) requestDelete(ctx context.Context, container *models.SContainer) {
t.SetStage("OnDeleted", nil)
if err := t.GetPodDriver().RequestDeleteContainer(ctx, t.GetUserCred(), t); err != nil {
t.OnDeleteFailed(ctx, container, jsonutils.NewString(err.Error()))
return
}
}
func (t *ContainerDeleteTask) OnDeleted(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
if err := container.RealDelete(ctx, t.GetUserCred()); err != nil {
t.OnDeleteFailed(ctx, container, jsonutils.NewString(errors.Wrap(err, "RealDelete").Error()))
return
}
t.SetStageComplete(ctx, nil)
}
func (t *ContainerDeleteTask) OnDeleteFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
container.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_DELETE_FAILED, reason.String())
t.SetStageFailed(ctx, reason)
}

56
pkg/compute/tasks/container_pull_image_task.go Normal file
View File

@@ -0,0 +1,56 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
)
func init() {
taskman.RegisterTask(ContainerPullImageTask{})
}
type ContainerPullImageTask struct {
ContainerBaseTask
}
func (t *ContainerPullImageTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
t.requestPullImage(ctx, obj.(*models.SContainer))
}
func (t *ContainerPullImageTask) requestPullImage(ctx context.Context, container *models.SContainer) {
t.SetStage("OnPulled", nil)
if err := t.GetPodDriver().RequestPullContainerImage(ctx, t.GetUserCred(), t); err != nil {
t.OnPulledFailed(ctx, container, jsonutils.NewString(err.Error()))
return
}
}
func (t *ContainerPullImageTask) OnPulledFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
container.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_PULL_IMAGE_FAILED, reason.String())
t.SetStageFailed(ctx, reason)
}
func (t *ContainerPullImageTask) OnPulled(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
container.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_PULLED_IMAGE, "")
t.SetStageComplete(ctx, nil)
}

64
pkg/compute/tasks/container_start_task.go Normal file
View File

@@ -0,0 +1,64 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
)
type ContainerStartTask struct {
ContainerBaseTask
}
func init() {
taskman.RegisterTask(ContainerStartTask{})
}
func (t *ContainerStartTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
t.requestStart(ctx, obj.(*models.SContainer))
}
func (t *ContainerStartTask) requestStart(ctx context.Context, container *models.SContainer) {
t.SetStage("OnStarted", nil)
if err := t.GetPodDriver().RequestStartContainer(ctx, t.GetUserCred(), t); err != nil {
t.OnStartedFailed(ctx, container, jsonutils.NewString(err.Error()))
return
}
}
func (t *ContainerStartTask) OnStarted(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
t.SetStage("OnSyncStatus", nil)
container.StartSyncStatusTask(ctx, t.GetUserCred(), t.GetTaskId())
}
func (t *ContainerStartTask) OnStartedFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
container.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_START_FAILED, reason.String())
t.SetStageFailed(ctx, reason)
}
func (t *ContainerStartTask) OnSyncStatus(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
t.SetStageComplete(ctx, nil)
}
func (t *ContainerStartTask) OnSyncStatusFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
t.SetStageFailed(ctx, reason)
}

62
pkg/compute/tasks/container_stop_task.go Normal file
View File

@@ -0,0 +1,62 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
)
type ContainerStopTask struct {
ContainerBaseTask
}
func init() {
taskman.RegisterTask(ContainerStopTask{})
}
func (t *ContainerStopTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
t.requestStop(ctx, obj.(*models.SContainer))
}
func (t *ContainerStopTask) requestStop(ctx context.Context, container *models.SContainer) {
t.SetStage("OnStopped", nil)
if err := t.GetPodDriver().RequestStopContainer(ctx, t.GetUserCred(), t); err != nil {
t.OnStoppedFailed(ctx, container, jsonutils.NewString(err.Error()))
return
}
}
func (t *ContainerStopTask) OnStoppedFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
container.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_STOP_FAILED, reason.String())
t.SetStageFailed(ctx, reason)
}
func (t *ContainerStopTask) OnStopped(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
t.SetStage("OnSyncStatus", nil)
container.StartSyncStatusTask(ctx, t.GetUserCred(), t.GetTaskId())
}
func (t *ContainerStopTask) OnSyncStatus(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
t.SetStageComplete(ctx, nil)
}
func (t *ContainerStopTask) OnSyncStatusFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
t.SetStageFailed(ctx, reason)
}

54
pkg/compute/tasks/container_sync_status_task.go Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
)
func init() {
taskman.RegisterTask(ContainerSyncStatusTask{})
}
type ContainerSyncStatusTask struct {
ContainerBaseTask
}
func (t *ContainerSyncStatusTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
t.SetStage("OnSyncStatus", nil)
if err := t.GetPodDriver().RequestSyncContainerStatus(ctx, t.GetUserCred(), t); err != nil {
t.OnSyncStatusFailed(ctx, obj.(*models.SContainer), jsonutils.NewString(err.Error()))
return
}
}
func (t *ContainerSyncStatusTask) OnSyncStatus(ctx context.Context, container *models.SContainer, data jsonutils.JSONObject) {
resp := new(api.ContainerSyncStatusResponse)
data.Unmarshal(resp)
container.SetStatus(ctx, t.GetUserCred(), resp.Status, "")
t.SetStageComplete(ctx, nil)
}
func (t *ContainerSyncStatusTask) OnSyncStatusFailed(ctx context.Context, container *models.SContainer, reason jsonutils.JSONObject) {
container.SetStatus(ctx, t.GetUserCred(), api.CONTAINER_STATUS_SYNC_STATUS_FAILED, reason.String())
t.SetStageFailed(ctx, reason)
}

116
pkg/compute/tasks/pod_create_task.go Normal file
View File

@@ -0,0 +1,116 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"fmt"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/sets"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
)
type PodCreateTask struct {
SGuestBaseTask
}
func init() {
taskman.RegisterTask(PodCreateTask{})
}
func (t *PodCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
t.SetStage("OnPodCreated", nil)
t.OnWaitPodCreated(ctx, obj, nil)
}
func (t *PodCreateTask) OnWaitPodCreated(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
task, err := taskman.TaskManager.NewTask(ctx, "GuestCreateTask", obj, t.GetUserCred(), t.GetParams(), t.GetTaskId(), "", nil)
if err != nil {
t.SetStageFailed(ctx, jsonutils.NewString(fmt.Sprintf("New GuestCreateTask")))
return
}
if err := task.ScheduleRun(nil); err != nil {
t.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}
}
func (t *PodCreateTask) OnPodCreated(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
t.SetStage("OnContainerCreated", nil)
guest.SetStatus(ctx, t.GetUserCred(), api.POD_STATUS_CREATING_CONTAINER, "")
ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId())
if err != nil {
t.onCreateContainerError(ctx, guest, errors.Wrapf(err, "get containers by pod %s", guest.GetId()))
return
}
for idx, ctr := range ctrs {
if err := ctr.StartCreateTask(ctx, t.GetUserCred(), t.GetTaskId()); err != nil {
t.onCreateContainerError(ctx, guest, errors.Wrapf(err, "start container %d creation task", idx))
return
}
}
}
func (t *PodCreateTask) onCreateContainerError(ctx context.Context, guest *models.SGuest, err error) {
guest.SetStatus(ctx, t.GetUserCred(), api.POD_STATUS_CREATE_CONTAINER_FAILED, err.Error())
t.onError(ctx, err)
}
func (t *PodCreateTask) onError(ctx context.Context, err error) {
t.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}
func (t *PodCreateTask) OnPodCreatedFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
t.SetStageFailed(ctx, data)
}
func (t *PodCreateTask) OnContainerCreated(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId())
if err != nil {
t.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
return
}
isAllCreated := true
createdStatus := []string{api.CONTAINER_STATUS_RUNNING, api.CONTAINER_STATUS_UNKNOWN, api.CONTAINER_STATUS_CREATED, api.CONTAINER_STATUS_EXITED}
for _, ctr := range ctrs {
if !sets.NewString(createdStatus...).Has(ctr.GetStatus()) {
isAllCreated = false
}
}
if isAllCreated {
t.SetStage("OnStatusSynced", nil)
guest.StartSyncstatus(ctx, t.GetUserCred(), t.GetTaskId())
}
}
func (t *PodCreateTask) OnContainerCreatedFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
guest.SetStatus(ctx, t.GetUserCred(), api.POD_STATUS_CREATE_CONTAINER_FAILED, data.String())
t.SetStageFailed(ctx, data)
}
func (t *PodCreateTask) OnStatusSynced(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
t.SetStageComplete(ctx, nil)
}
func (t *PodCreateTask) OnStatusSyncedFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
t.SetStageFailed(ctx, data)
}

84
pkg/compute/tasks/pod_delete_task.go Normal file
View File

@@ -0,0 +1,84 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/guestdrivers"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/httperrors"
)
type PodDeleteTask struct {
SGuestBaseTask
}
func init() {
taskman.RegisterTask(PodDeleteTask{})
}
func (t *PodDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
t.SetStage("OnWaitContainerDeleted", nil)
t.OnWaitContainerDeleted(ctx, obj.(*models.SGuest), nil)
}
func (t *PodDeleteTask) OnWaitContainerDeleted(ctx context.Context, pod *models.SGuest, _ jsonutils.JSONObject) {
pod.SetStatus(ctx, t.GetUserCred(), api.POD_STATUS_DELETING_CONTAINER, "")
ctrs, err := models.GetContainerManager().GetContainersByPod(pod.GetId())
if err != nil {
t.OnWaitContainerDeletedFailed(ctx, pod, jsonutils.NewString(errors.Wrap(err, "GetContainersByPod").Error()))
return
}
if len(ctrs) == 0 {
t.OnContainerDeleted(ctx, pod)
return
}
curCtr := ctrs[0]
curCtr.StartDeleteTask(ctx, t.GetUserCred(), t.GetTaskId())
}
func (t *PodDeleteTask) OnWaitContainerDeletedFailed(ctx context.Context, pod *models.SGuest, data jsonutils.JSONObject) {
pod.SetStatus(ctx, t.GetUserCred(), api.POD_STATUS_DELETE_CONTAINER_FAILED, data.String())
t.SetStageFailed(ctx, data)
}
func (t *PodDeleteTask) OnContainerDeleted(ctx context.Context, pod *models.SGuest) {
t.SetStage("OnPodUndeploy", nil)
host, _ := pod.GetHost()
if err := pod.GetDriver().(*guestdrivers.SPodDriver).RequestUndeployPod(ctx, pod, host, t); err != nil {
if errors.Cause(err) == httperrors.ErrNotFound {
t.OnPodUndeploy(ctx, pod, nil)
return
}
t.OnPodUndeployFailed(ctx, pod, jsonutils.NewString(err.Error()))
return
}
}
func (t *PodDeleteTask) OnPodUndeploy(ctx context.Context, pod *models.SGuest, data jsonutils.JSONObject) {
t.SetStageComplete(ctx, nil)
}
func (t *PodDeleteTask) OnPodUndeployFailed(ctx context.Context, pod *models.SGuest, reason jsonutils.JSONObject) {
pod.SetStatus(ctx, t.GetUserCred(), api.VM_DELETE_FAIL, reason.String())
t.SetStageFailed(ctx, reason)
}

4
pkg/compute/usages/handler.go
View File

@@ -999,7 +999,7 @@ func guestHypervisorsUsage(
count[fmt.Sprintf("%s.cpu", prefix)] = guest.TotalCpuCount
count[fmt.Sprintf("%s.memory", prefix)] = guest.TotalMemSize
if len(hypervisors) == 1 && hypervisors[0] == api.HYPERVISOR_CONTAINER {
if len(hypervisors) == 1 && hypervisors[0] == api.HYPERVISOR_POD {
return count
}
@@ -1020,7 +1020,7 @@ func guestUsage(ctx context.Context, userToken mcclient.TokenCredential, prefix
policyResult rbacutils.SPolicyResult,
) Usage {
hypervisors := sets.NewString(api.HYPERVISORS...)
hypervisors.Delete(api.HYPERVISOR_CONTAINER, api.HYPERVISOR_BAREMETAL)
hypervisors.Delete(api.HYPERVISOR_POD, api.HYPERVISOR_BAREMETAL)
return guestHypervisorsUsage(ctx, userToken, prefix, scope, userCred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, status, hypervisors.List(), pendingDelete, includeSystem, since, policyResult)
}

1
pkg/hostman/container/device/doc.go Normal file
View File

@@ -0,0 +1 @@
package device // import "yunion.io/x/onecloud/pkg/hostman/container/device"

33
pkg/hostman/container/device/host.go Normal file
View File

@@ -0,0 +1,33 @@
package device
import (
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/onecloud/pkg/apis"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
)
func init() {
RegisterDriver(newHostDevice())
}
type hostDevice struct {
}
func newHostDevice() IDeviceDriver {
return &hostDevice{}
}
func (h hostDevice) GetType() apis.ContainerDeviceType {
return apis.CONTAINER_DEVICE_TYPE_HOST
}
func (h hostDevice) GetRuntimeDevices(_ *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) {
return []*runtimeapi.Device{
{
ContainerPath: dev.ContainerPath,
HostPath: dev.Host.HostPath,
Permissions: dev.Permissions,
},
}, nil
}

31
pkg/hostman/container/device/interface.go Normal file
View File

@@ -0,0 +1,31 @@
package device
import (
"fmt"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/onecloud/pkg/apis"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
)
var (
drivers = make(map[apis.ContainerDeviceType]IDeviceDriver)
)
func RegisterDriver(drv IDeviceDriver) {
drivers[drv.GetType()] = drv
}
func GetDriver(typ apis.ContainerDeviceType) IDeviceDriver {
drv, ok := drivers[typ]
if !ok {
panic(fmt.Sprintf("not found driver by type %s", typ))
}
return drv
}
type IDeviceDriver interface {
GetType() apis.ContainerDeviceType
GetRuntimeDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error)
}

37
pkg/hostman/container/device/isolated_device.go Normal file
View File

@@ -0,0 +1,37 @@
package device
import (
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/apis"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
)
func init() {
RegisterDriver(newIsolatedDevice())
}
type isolatedDevice struct{}
func newIsolatedDevice() IDeviceDriver {
return &isolatedDevice{}
}
func (i isolatedDevice) GetType() apis.ContainerDeviceType {
return apis.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE
}
func (i isolatedDevice) GetRuntimeDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) {
man, err := isolated_device.GetContainerDeviceManager(isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType))
if err != nil {
return nil, errors.Wrapf(err, "GetContainerDeviceManager by type %q", dev.Type)
}
ctrDevs, err := man.NewContainerDevices(input, dev)
if err != nil {
return nil, errors.Wrapf(err, "NewContainerDevices with %#v", dev)
}
return ctrDevs, nil
}

1
pkg/hostman/container/storage/doc.go Normal file
View File

@@ -0,0 +1 @@
package storage // import "yunion.io/x/onecloud/pkg/hostman/container/storage"

88
pkg/hostman/container/storage/local_raw.go Normal file
View File

@@ -0,0 +1,88 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"fmt"
losetup "github.com/zexi/golosetup"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/util/fileutils2"
)
func init() {
RegisterDriver(newLocalRaw())
}
type localRaw struct{}
func newLocalRaw() *localRaw {
return &localRaw{}
}
func (l localRaw) GetType() StorageType {
return STORAGE_TYPE_LOCAL_RAW
}
func (l localRaw) CheckConnect(diskPath string) (string, bool, error) {
devs, err := losetup.ListDevices()
if err != nil {
return "", false, errors.Wrap(err, "list loop devices")
}
for _, dev := range devs.LoopDevs {
if dev.BackFile == diskPath {
return l.checkPartition(dev.Name), true, nil
}
}
return "", false, nil
}
func (l localRaw) ConnectDisk(diskPath string) (string, error) {
loDev, err := losetup.AttachDevice(diskPath, true)
if err != nil {
return "", errors.Wrapf(err, "failed to attach %s as loop device", diskPath)
}
return l.checkPartition(loDev.Name), nil
}
func (l localRaw) checkPartition(devName string) string {
partPath := fmt.Sprintf("%sp1", devName)
if fileutils2.Exists(partPath) {
return partPath
}
return devName
}
func (l localRaw) DisconnectDisk(diskPath string, mountPoint string) error {
devs, err := losetup.ListDevices()
if err != nil {
return errors.Wrap(err, "list loop devices")
}
for _, dev := range devs.LoopDevs {
if dev.BackFile == diskPath {
log.Infof("Start detach loop device %s", dev.Name)
if err := losetup.DetachDevice(dev.Name); err != nil {
return errors.Wrapf(err, "detach device %s", dev.Name)
} else {
log.Infof("detach loop device %s of disk %s", dev.Name, diskPath)
return nil
}
}
}
return nil
}

91
pkg/hostman/container/storage/storage.go Normal file
View File

@@ -0,0 +1,91 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"context"
"fmt"
"strings"
"time"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
)
var (
drivers = make(map[StorageType]IContainerStorage)
)
type StorageType string
const (
STORAGE_TYPE_LOCAL_RAW StorageType = "local_raw"
STORAGE_TYPE_LOCAL_QCOW2 StorageType = "local_qcow2"
)
type IContainerStorage interface {
GetType() StorageType
CheckConnect(diskPath string) (string, bool, error)
ConnectDisk(diskPath string) (string, error)
DisconnectDisk(diskPath string, mountPoint string) error
}
func GetDriver(t StorageType) IContainerStorage {
return drivers[t]
}
func RegisterDriver(drv IContainerStorage) {
_, ok := drivers[drv.GetType()]
if ok {
panic(fmt.Sprintf("driver %s already registered", drv.GetType()))
}
drivers[drv.GetType()] = drv
}
func Mount(devPath string, mountPoint string, fsType string) error {
if !fileutils2.Exists(mountPoint) {
output, err := procutils.NewCommand("mkdir", "-p", mountPoint).Output()
if err != nil {
return errors.Wrapf(err, "mkdir %s failed: %s", mountPoint, output)
}
}
if err := procutils.NewRemoteCommandAsFarAsPossible("mountpoint", mountPoint).Run(); err == nil {
log.Warningf("mountpoint %s is already mounted", mountPoint)
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", "-t", fsType, devPath, mountPoint).Output(); err != nil {
return errors.Wrapf(err, "mount %s to %s with fs %s: %s", devPath, mountPoint, fsType, string(out))
}
return nil
}
func Unmount(mountPoint string) error {
mountOut, err := procutils.NewRemoteCommandAsFarAsPossible("mountpoint", mountPoint).Output()
if err == nil {
out, err := procutils.NewRemoteCommandAsFarAsPossible("umount", mountPoint).Output()
if err != nil {
return errors.Wrapf(err, "umount %s failed %s", mountPoint, out)
}
}
if strings.Contains(string(mountOut), "No such file or directory") {
return nil
}
return errors.Wrapf(err, "check mountpoint %s: %s", mountPoint, string(mountOut))
}

162
pkg/hostman/container/volume_mount/disk.go Normal file
View File

@@ -0,0 +1,162 @@
package volume_mount
import (
"fmt"
"path/filepath"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/apis"
"yunion.io/x/onecloud/pkg/hostman/container/storage"
container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage"
"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
"yunion.io/x/onecloud/pkg/hostman/storageman"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/util/procutils"
)
func init() {
RegisterDriver(newDisk())
}
type disk struct{}
func newDisk() IVolumeMount {
return &disk{}
}
func (d disk) GetType() apis.ContainerVolumeMountType {
return apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK
}
func (d disk) GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) {
diskInput := vm.Disk
if diskInput == nil {
return "", httperrors.NewNotEmptyError("disk is nil")
}
hostPath := filepath.Join(pod.GetVolumesDir(), diskInput.Id)
if diskInput.SubDirectory != "" {
return filepath.Join(hostPath, diskInput.SubDirectory), nil
}
if diskInput.StorageSizeFile != "" {
return filepath.Join(hostPath, diskInput.StorageSizeFile), nil
}
return hostPath, nil
}
func (d disk) getPodDisk(pod IPodInfo, vm *apis.ContainerVolumeMount) (storageman.IDisk, *desc.SGuestDisk, error) {
var disk *desc.SGuestDisk = nil
disks := pod.GetDisks()
volDisk := vm.Disk
if volDisk.Id == "" {
return nil, nil, errors.Errorf("volume mount disk id is empty")
}
if volDisk.Id != "" {
for _, gd := range disks {
if gd.DiskId == volDisk.Id {
disk = gd
break
}
}
}
if disk == nil {
return nil, nil, errors.Wrapf(errors.ErrNotFound, "not found disk by id %s", volDisk.Id)
}
iDisk, err := storageman.GetManager().GetDiskById(disk.DiskId)
if err != nil {
return nil, disk, errors.Wrapf(err, "GetDiskById %s", disk.Path)
}
return iDisk, disk, nil
}
func (d disk) getDiskStorageDriver(pod IPodInfo, vm *apis.ContainerVolumeMount) (storage.IContainerStorage, error) {
iDisk, _, err := d.getPodDisk(pod, vm)
if err != nil {
return nil, errors.Wrap(err, "get pod disk interface")
}
drv, err := iDisk.GetContainerStorageDriver()
if err != nil {
return nil, errors.Wrap(err, "GetContainerStorageDriver")
}
return drv, nil
}
func (d disk) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
iDisk, gd, err := d.getPodDisk(pod, vm)
if err != nil {
return errors.Wrap(err, "get pod disk interface")
}
drv, err := iDisk.GetContainerStorageDriver()
if err != nil {
return errors.Wrap(err, "get disk storage driver")
}
devPath, isConnected, err := drv.CheckConnect(iDisk.GetPath())
if err != nil {
return errors.Wrapf(err, "CheckConnect %s", iDisk.GetPath())
}
log.Infof("=======check connect: %q %q %v", iDisk.GetPath(), devPath, isConnected)
if !isConnected {
devPath, err = drv.ConnectDisk(iDisk.GetPath())
if err != nil {
return errors.Wrapf(err, "ConnectDisk %s", iDisk.GetPath())
}
}
mntPoint := pod.GetDiskMountPoint(iDisk)
if err := container_storage.Mount(devPath, mntPoint, gd.Fs); err != nil {
return errors.Wrapf(err, "mount %s to %s", devPath, mntPoint)
}
vmDisk := vm.Disk
if vmDisk.SubDirectory != "" {
out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", filepath.Join(mntPoint, vmDisk.SubDirectory)).Output()
if err != nil {
return errors.Wrapf(err, "make sub_directory %s inside %s: %s", vmDisk.SubDirectory, mntPoint, out)
}
}
if vmDisk.StorageSizeFile != "" {
if err := d.createStorageSizeFile(iDisk, mntPoint, vmDisk); err != nil {
return errors.Wrapf(err, "create storage file %s inside %s", vmDisk.StorageSizeFile, mntPoint)
}
}
return nil
}
func (d disk) createStorageSizeFile(iDisk storageman.IDisk, mntPoint string, input *apis.ContainerVolumeMountDisk) error {
desc := iDisk.GetDiskDesc()
diskSizeMB, err := desc.Int("disk_size")
if err != nil {
return errors.Wrapf(err, "get disk_size from %s", desc.String())
}
sp := filepath.Join(mntPoint, input.StorageSizeFile)
sizeBytes := diskSizeMB * 1024
out, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", fmt.Sprintf("echo %d > %s", sizeBytes, sp)).Output()
if err != nil {
return errors.Wrapf(err, "write %d to %s: %s", sizeBytes, sp, out)
}
return nil
}
func (d disk) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
iDisk, _, err := d.getPodDisk(pod, vm)
if err != nil {
return errors.Wrap(err, "get pod disk interface")
}
drv, err := iDisk.GetContainerStorageDriver()
if err != nil {
return errors.Wrap(err, "get disk storage driver")
}
mntPoint := pod.GetDiskMountPoint(iDisk)
if err := container_storage.Unmount(mntPoint); err != nil {
return errors.Wrapf(err, "unmount %s", mntPoint)
}
_, isConnected, err := drv.CheckConnect(iDisk.GetPath())
if err != nil {
return errors.Wrapf(err, "CheckConnect %s", iDisk.GetPath())
}
if isConnected {
if err := drv.DisconnectDisk(iDisk.GetPath(), mntPoint); err != nil {
return errors.Wrapf(err, "DisconnectDisk %s %s", iDisk.GetPath(), mntPoint)
}
}
return nil
}

1
pkg/hostman/container/volume_mount/doc.go Normal file
View File

@@ -0,0 +1 @@
package volume_mount // import "yunion.io/x/onecloud/pkg/hostman/container/volume_mount"

36
pkg/hostman/container/volume_mount/host_path.go Normal file
View File

@@ -0,0 +1,36 @@
package volume_mount
import (
"yunion.io/x/onecloud/pkg/apis"
"yunion.io/x/onecloud/pkg/httperrors"
)
func init() {
RegisterDriver(newHostLocal())
}
type hostLocal struct{}
func (h hostLocal) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
return nil
}
func (h hostLocal) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
return nil
}
func newHostLocal() IVolumeMount {
return &hostLocal{}
}
func (h hostLocal) GetType() apis.ContainerVolumeMountType {
return apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH
}
func (h hostLocal) GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) {
host := vm.HostPath
if host == nil {
return "", httperrors.NewNotEmptyError("host_local is nil")
}
return host.Path, nil
}

53
pkg/hostman/container/volume_mount/interface.go Normal file
View File

@@ -0,0 +1,53 @@
package volume_mount
import (
"fmt"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/onecloud/pkg/apis"
"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
"yunion.io/x/onecloud/pkg/hostman/storageman"
)
var (
drivers = make(map[apis.ContainerVolumeMountType]IVolumeMount)
)
func RegisterDriver(drv IVolumeMount) {
drivers[drv.GetType()] = drv
}
func GetDriver(typ apis.ContainerVolumeMountType) IVolumeMount {
drv, ok := drivers[typ]
if !ok {
panic(fmt.Sprintf("not found driver by type %s", typ))
}
return drv
}
type IPodInfo interface {
GetVolumesDir() string
GetDisks() []*desc.SGuestDisk
GetDiskMountPoint(disk storageman.IDisk) string
}
type IVolumeMount interface {
GetType() apis.ContainerVolumeMountType
GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error)
Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error
Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error
}
func GetRuntimeVolumeMountPropagation(input apis.ContainerMountPropagation) runtimeapi.MountPropagation {
switch input {
case apis.MOUNTPROPAGATION_PROPAGATION_PRIVATE:
return runtimeapi.MountPropagation_PROPAGATION_PRIVATE
case apis.MOUNTPROPAGATION_PROPAGATION_HOST_TO_CONTAINER:
return runtimeapi.MountPropagation_PROPAGATION_HOST_TO_CONTAINER
case apis.MOUNTPROPAGATION_PROPAGATION_BIDIRECTIONAL:
return runtimeapi.MountPropagation_PROPAGATION_BIDIRECTIONAL
}
// private defaultly
return runtimeapi.MountPropagation_PROPAGATION_PRIVATE
}

8
pkg/hostman/guestman/desc/desc.go
View File

@@ -363,7 +363,8 @@ type SGuestControlDesc struct {
EncryptKeyId string
LightMode bool // light mode
LightMode bool // light mode
Hypervisor string
}
type SGuestMetaDesc struct {
@@ -379,10 +380,15 @@ type SGuestMetaDesc struct {
ExtraOptions map[string]jsonutils.JSONObject
}
type SGuestContainerDesc struct {
Containers []*api.ContainerDesc
}
type SGuestDesc struct {
SGuestProjectDesc
SGuestRegionDesc
SGuestControlDesc
SGuestHardwareDesc
SGuestMetaDesc
SGuestContainerDesc
}

2
pkg/hostman/guestman/guest-agent.go
View File

@@ -25,7 +25,7 @@ import (
)
func (m *SGuestManager) checkAndInitGuestQga(sid string) (*SKVMGuestInstance, error) {
guest, _ := m.GetServer(sid)
guest, _ := m.GetKVMServer(sid)
if guest == nil {
return nil, httperrors.NewNotFoundError("Not found guest by id %s", sid)
}

8
pkg/hostman/guestman/guest_create_from_remote.go
View File

@@ -36,8 +36,8 @@ func (m *SGuestManager) GuestCreateFromEsxi(
if !ok {
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(createConfig.Sid)
if err := guest.SaveDesc(createConfig.GuestDesc); err != nil {
guest, _ := m.GetKVMServer(createConfig.Sid)
if err := SaveDesc(guest, createConfig.GuestDesc); err != nil {
return nil, err
}
@@ -121,8 +121,8 @@ func (m *SGuestManager) GuestCreateFromCloudpods(
if !ok {
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(createConfig.Sid)
if err := guest.SaveDesc(createConfig.GuestDesc); err != nil {
guest, _ := m.GetKVMServer(createConfig.Sid)
if err := SaveDesc(guest, createConfig.GuestDesc); err != nil {
return nil, err
}
var err error

12
pkg/hostman/guestman/guesthandlers/guesthandler.go
View File

@@ -141,7 +141,7 @@ func guestActions(f actionFunc) appsrv.FilterHandler {
func getStatus(ctx context.Context, w http.ResponseWriter, r *http.Request) {
params, _, _ := appsrv.FetchEnv(ctx, w, r)
sid := params["<sid>"]
hostutils.DelayTaskWithoutReqctx(ctx, guestman.GetGuestManager().StatusWithBlockJobsCount, sid)
hostutils.DelayTaskWithoutReqctx(ctx, guestman.GetGuestManager().GetGuestStatus, sid)
hostutils.ResponseOk(ctx, w)
}
@@ -486,7 +486,7 @@ func guestLiveMigrate(ctx context.Context, userCred mcclient.TokenCredential, si
}
func guestCancelLiveMigrate(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (interface{}, error) {
guest, ok := guestman.GetGuestManager().GetServer(sid)
guest, ok := guestman.GetGuestManager().GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("Guest %s not found", sid)
}
@@ -545,7 +545,7 @@ func guestBlockReplication(ctx context.Context, userCred mcclient.TokenCredentia
}
func slaveGuestBlockStreamDisks(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (interface{}, error) {
guest, ok := guestman.GetGuestManager().GetServer(sid)
guest, ok := guestman.GetGuestManager().GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("Guest %s not found", sid)
}
@@ -595,7 +595,7 @@ func guestReloadDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredent
if err != nil {
return nil, httperrors.NewMissingParameterError("disk_id")
}
guest, ok := guestman.GetGuestManager().GetServer(sid)
guest, ok := guestman.GetGuestManager().GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("guest %s not found", sid)
}
@@ -628,7 +628,7 @@ func guestSnapshot(ctx context.Context, userCred mcclient.TokenCredential, sid s
if err != nil {
return nil, httperrors.NewMissingParameterError("disk_id")
}
guest, ok := guestman.GetGuestManager().GetServer(sid)
guest, ok := guestman.GetGuestManager().GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("guest %s not found", sid)
}
@@ -667,7 +667,7 @@ func guestDeleteSnapshot(ctx context.Context, userCred mcclient.TokenCredential,
if err != nil {
return nil, httperrors.NewMissingParameterError("disk_id")
}
guest, ok := guestman.GetGuestManager().GetServer(sid)
guest, ok := guestman.GetGuestManager().GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("guest %s not found", sid)
}

323
pkg/hostman/guestman/guestman.go
View File

@@ -56,9 +56,9 @@ import (
"yunion.io/x/onecloud/pkg/mcclient"
modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
"yunion.io/x/onecloud/pkg/util/cgrouputils"
"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/netutils2"
"yunion.io/x/onecloud/pkg/util/pod"
"yunion.io/x/onecloud/pkg/util/procutils"
"yunion.io/x/onecloud/pkg/util/timeutils2"
)
@@ -84,7 +84,7 @@ type SGuestManager struct {
host hostutils.IHost
ServersPath string
Servers *sync.Map
CandidateServers map[string]*SKVMGuestInstance
CandidateServers map[string]GuestRuntimeInstance
UnknownServers *sync.Map
ServersLock *sync.Mutex
portsInUse *sync.Map
@@ -97,7 +97,7 @@ type SGuestManager struct {
isLoaded bool
// dirty servers chan
dirtyServers []*SKVMGuestInstance
dirtyServers []GuestRuntimeInstance
dirtyServersChan chan struct{}
qemuMachineCpuMax map[string]uint
@@ -114,7 +114,7 @@ func NewGuestManager(host hostutils.IHost, serversPath string) (*SGuestManager,
manager.ServersPath = serversPath
manager.Servers = new(sync.Map)
manager.portsInUse = new(sync.Map)
manager.CandidateServers = make(map[string]*SKVMGuestInstance, 0)
manager.CandidateServers = make(map[string]GuestRuntimeInstance, 0)
manager.UnknownServers = new(sync.Map)
manager.ServersLock = &sync.Mutex{}
manager.TrafficLock = &sync.Mutex{}
@@ -122,7 +122,7 @@ func NewGuestManager(host hostutils.IHost, serversPath string) (*SGuestManager,
// manager.StartCpusetBalancer()
manager.dirtyServersChan = make(chan struct{})
manager.dirtyServers = make([]*SKVMGuestInstance, 0)
manager.dirtyServers = make([]GuestRuntimeInstance, 0)
manager.qemuMachineCpuMax = make(map[string]uint, 0)
err := procutils.NewCommand("mkdir", "-p", manager.QemuLogDir()).Run()
if err != nil {
@@ -202,6 +202,10 @@ func (m *SGuestManager) InitPythonPath() error {
return errors.Errorf("No python/python2/python3 found in PATH")
}
func (m *SGuestManager) GetCRI() pod.CRI {
return m.host.GetCRI()
}
func (m *SGuestManager) getPythonPath() string {
return m.pythonPath
}
@@ -210,25 +214,34 @@ func (m *SGuestManager) QemuLogDir() string {
return path.Join(m.ServersPath, "logs")
}
func (m *SGuestManager) GetServer(sid string) (*SKVMGuestInstance, bool) {
func (m *SGuestManager) GetServer(sid string) (GuestRuntimeInstance, bool) {
s, ok := m.Servers.Load(sid)
if ok {
return s.(*SKVMGuestInstance), ok
return s.(GuestRuntimeInstance), ok
} else {
return nil, ok
}
}
func (m *SGuestManager) GetUnknownServer(sid string) (*SKVMGuestInstance, bool) {
// 临时解决方案，后面应该统一 SKVMInstance 和 SPodInstance 使用 GuestRuntimeInstance 接口
func (m *SGuestManager) GetKVMServer(sid string) (*SKVMGuestInstance, bool) {
s, ok := m.GetServer(sid)
if !ok {
return nil, false
}
return s.(*SKVMGuestInstance), true
}
func (m *SGuestManager) GetUnknownServer(sid string) (GuestRuntimeInstance, bool) {
s, ok := m.UnknownServers.Load(sid)
if ok {
return s.(*SKVMGuestInstance), ok
return s.(GuestRuntimeInstance), ok
} else {
return nil, ok
}
}
func (m *SGuestManager) SaveServer(sid string, s *SKVMGuestInstance) {
func (m *SGuestManager) SaveServer(sid string, s GuestRuntimeInstance) {
m.Servers.Store(sid, s)
}
@@ -313,9 +326,9 @@ func (m *SGuestManager) OnVerifyExistingGuestsSucc(servers []jsonutils.JSONObjec
}
}
func (m *SGuestManager) RemoveCandidateServer(server *SKVMGuestInstance) {
if _, ok := m.CandidateServers[server.Id]; ok {
delete(m.CandidateServers, server.Id)
func (m *SGuestManager) RemoveCandidateServer(server GuestRuntimeInstance) {
if _, ok := m.CandidateServers[server.GetInitialId()]; ok {
delete(m.CandidateServers, server.GetInitialId())
if len(m.CandidateServers) == 0 {
m.OnLoadExistingGuestsComplete()
}
@@ -382,7 +395,7 @@ func (m *SGuestManager) cpusetBalance() {
}
func (m *SGuestManager) CPUSet(ctx context.Context, sid string, req *compute.ServerCPUSetInput) (*compute.ServerCPUSetResp, error) {
guest, ok := m.GetServer(sid)
guest, ok := m.GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("Not found")
}
@@ -390,7 +403,7 @@ func (m *SGuestManager) CPUSet(ctx context.Context, sid string, req *compute.Ser
}
func (m *SGuestManager) CPUSetRemove(ctx context.Context, sid string) error {
guest, ok := m.GetServer(sid)
guest, ok := m.GetKVMServer(sid)
if !ok {
return httperrors.NewNotFoundError("Not found")
}
@@ -422,60 +435,43 @@ func (m *SGuestManager) LoadExistingGuests() {
}
}
func (m *SGuestManager) LoadServer(sid string) {
guest := NewKVMGuestInstance(sid, m)
err := guest.LoadDesc()
func (m *SGuestManager) GetServerDescFilePath(sid string) string {
return path.Join(m.ServersPath, sid, "desc")
}
func (m *SGuestManager) GetServerDesc(sid string) (*desc.SGuestDesc, error) {
descPath := m.GetServerDescFilePath(sid)
descStr, err := ioutil.ReadFile(descPath)
if err != nil {
return nil, errors.Wrapf(err, "read file %s", descPath)
}
desc := new(desc.SGuestDesc)
jsonSrcDesc, err := jsonutils.Parse(descStr)
if err != nil {
return nil, errors.Wrapf(err, "json parse: %s", descStr)
}
if err := jsonSrcDesc.Unmarshal(desc); err != nil {
return nil, errors.Wrap(err, "unmarshal desc")
}
return desc, nil
}
func (m *SGuestManager) LoadServer(sid string) {
desc, err := m.GetServerDesc(sid)
if err != nil {
log.Errorf("Get server %s desc: %v", sid, err)
return
}
guest := NewGuestRuntimeManager().NewRuntimeInstance(sid, m, desc.Hypervisor)
if err := guest.LoadDesc(); err != nil {
log.Errorf("On load server error: %s", err)
return
}
if guest.needSyncStreamDisks {
go guest.sendStreamDisksComplete(context.Background())
}
m.CandidateServers[sid] = guest
m.loadGuestCpuset(guest)
}
func (m *SGuestManager) loadGuestCpuset(guest *SKVMGuestInstance) {
if guest.GetPid() > 0 {
m.cpuSet.Lock.Lock()
defer m.cpuSet.Lock.Unlock()
for _, vcpuPin := range guest.Desc.VcpuPin {
pcpuSet, err := cpuset.Parse(vcpuPin.Pcpus)
if err != nil {
log.Errorf("failed parse %s pcpus: %s", guest.GetName(), vcpuPin.Pcpus)
continue
}
vcpuSet, err := cpuset.Parse(vcpuPin.Vcpus)
if err != nil {
log.Errorf("failed parse %s vcpus: %s", guest.GetName(), vcpuPin.Vcpus)
continue
}
m.cpuSet.LoadCpus(pcpuSet.ToSlice(), vcpuSet.Size())
}
for _, numaCpuset := range guest.Desc.CpuNumaPin {
pcpuSet, err := cpuset.Parse(*numaCpuset.Pcpus)
if err != nil {
log.Errorf("failed parse %s pcpus: %s", guest.GetName(), *numaCpuset.Pcpus)
continue
}
vcpuCount := int(guest.Desc.Cpu)
if numaCpuset.Vcpus != nil {
vcpuSet, err := cpuset.Parse(*numaCpuset.Vcpus)
if err != nil {
log.Errorf("failed parse %s vcpus: %s", guest.GetName(), *numaCpuset.Vcpus)
continue
}
vcpuCount = vcpuSet.Size()
}
hostNodes := -1
if numaCpuset.HostNodes != nil {
hostNodes = int(*numaCpuset.HostNodes)
}
m.cpuSet.LoadNumaCpus(numaCpuset.SizeMB, hostNodes, pcpuSet.ToSlice(), vcpuCount)
}
if err := guest.PostLoad(m); err != nil {
log.Errorf("Post load server %s: %v", sid, err)
return
}
}
@@ -501,11 +497,11 @@ func (m *SGuestManager) GetGuestNicDesc(
var nic *desc.SGuestNetwork
var guestDesc *desc.SGuestDesc
m.Servers.Range(func(k interface{}, v interface{}) bool {
guest := v.(*SKVMGuestInstance)
guest := v.(GuestRuntimeInstance)
if guest.IsLoaded() {
nic = guest.GetNicDescMatch(mac, ip, port, bridge)
if nic != nil {
guestDesc = guest.Desc
guestDesc = guest.GetDesc()
return false
}
}
@@ -521,7 +517,7 @@ func (m *SGuestManager) getGuestNicDescInCandidate(
if guest.IsLoaded() {
nic := guest.GetNicDescMatch(mac, ip, port, bridge)
if nic != nil {
return guest.Desc, nic
return guest.GetDesc(), nic
}
}
}
@@ -536,7 +532,7 @@ func (m *SGuestManager) PrepareCreate(sid string) error {
}
guest := NewKVMGuestInstance(sid, m)
m.SaveServer(sid, guest)
return guest.PrepareDir()
return PrepareDir(guest)
}
func (m *SGuestManager) PrepareDeploy(sid string) error {
@@ -551,7 +547,7 @@ func (m *SGuestManager) PrepareDeploy(sid string) error {
}
func (m *SGuestManager) Monitor(sid, cmd string, qmp bool, callback func(string)) error {
if guest, ok := m.GetServer(sid); ok {
if guest, ok := m.GetKVMServer(sid); ok {
if guest.IsRunning() {
if guest.Monitor == nil {
return httperrors.NewBadRequestError("Monitor disconnected??")
@@ -568,7 +564,7 @@ func (m *SGuestManager) Monitor(sid, cmd string, qmp bool, callback func(string)
return httperrors.NewBadRequestError("Server stopped??")
}
} else {
return httperrors.NewNotFoundError("Not found")
return httperrors.NewNotFoundError("Not found KVM server: %s", sid)
}
}
@@ -582,7 +578,7 @@ func (m *SGuestManager) sdnClient() (fwdpb.ForwarderClient, error) {
}
func (m *SGuestManager) OpenForward(ctx context.Context, sid string, req *hostapi.GuestOpenForwardRequest) (*hostapi.GuestOpenForwardResponse, error) {
guest, ok := m.GetServer(sid)
guest, ok := m.GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("Not found")
}
@@ -636,7 +632,7 @@ func (m *SGuestManager) OpenForward(ctx context.Context, sid string, req *hostap
}
func (m *SGuestManager) CloseForward(ctx context.Context, sid string, req *hostapi.GuestCloseForwardRequest) (*hostapi.GuestCloseForwardResponse, error) {
guest, ok := m.GetServer(sid)
guest, ok := m.GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("Not found")
}
@@ -674,7 +670,7 @@ func (m *SGuestManager) CloseForward(ctx context.Context, sid string, req *hosta
}
func (m *SGuestManager) ListForward(ctx context.Context, sid string, req *hostapi.GuestListForwardRequest) (*hostapi.GuestListForwardResponse, error) {
guest, ok := m.GetServer(sid)
guest, ok := m.GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("Not found")
}
@@ -729,23 +725,31 @@ func (m *SGuestManager) GuestCreate(ctx context.Context, params interface{}) (js
return nil, hostutils.ParamsError
}
var guest *SKVMGuestInstance
var guest GuestRuntimeInstance
e := func() error {
m.ServersLock.Lock()
defer m.ServersLock.Unlock()
if _, ok := m.GetServer(deployParams.Sid); ok {
return httperrors.NewBadRequestError("Guest %s exists", deployParams.Sid)
}
guest = NewKVMGuestInstance(deployParams.Sid, m)
var (
descInfo *desc.SGuestDesc = nil
hypervisor = ""
)
if deployParams.Body.Contains("desc") {
var desc = new(desc.SGuestDesc)
err := deployParams.Body.Unmarshal(desc, "desc")
descInfo = new(desc.SGuestDesc)
err := deployParams.Body.Unmarshal(descInfo, "desc")
if err != nil {
return httperrors.NewBadRequestError("Guest desc unmarshal failed %s", err)
}
err = guest.CreateFromDesc(desc)
if err != nil {
hypervisor = descInfo.Hypervisor
}
//guest = NewKVMGuestInstance(deployParams.Sid, m)
factory := NewGuestRuntimeManager()
guest = factory.NewRuntimeInstance(deployParams.Sid, m, hypervisor)
if descInfo != nil {
if err := factory.CreateFromDesc(guest, descInfo); err != nil {
return errors.Wrap(err, "create from desc")
}
}
@@ -760,11 +764,7 @@ func (m *SGuestManager) GuestCreate(ctx context.Context, params interface{}) (js
}
func (m *SGuestManager) startDeploy(
ctx context.Context, deployParams *SGuestDeploy, guest *SKVMGuestInstance) (jsonutils.JSONObject, error) {
if jsonutils.QueryBoolean(deployParams.Body, "k8s_pod", false) {
return nil, nil
}
ctx context.Context, deployParams *SGuestDeploy, guest GuestRuntimeInstance) (jsonutils.JSONObject, error) {
publicKey := deployapi.GetKeys(deployParams.Body)
deployArray := make([]*deployapi.DeployContent, 0)
if deployParams.Body.Contains("deploys") {
@@ -792,7 +792,7 @@ func (m *SGuestManager) startDeploy(
password, deployParams.IsInit, false,
options.HostOptions.LinuxDefaultRootUser, options.HostOptions.WindowsDefaultAdminUser,
enableCloudInit, loginAccount, deployTelegraf, telegrafConfig,
guest.Desc.UserData,
guest.GetDesc().UserData,
),
)
if err != nil {
@@ -817,7 +817,7 @@ func (m *SGuestManager) GuestDeploy(ctx context.Context, params interface{}) (js
if err != nil {
return nil, httperrors.NewBadRequestError("Failed unmarshal guest desc %s", err)
}
if err := guest.SaveDesc(guestDesc); err != nil {
if err := SaveDesc(guest, guestDesc); err != nil {
return nil, errors.Wrap(err, "failed save desc")
}
}
@@ -838,42 +838,16 @@ func (m *SGuestManager) Status(sid string) string {
return status
}
func (m *SGuestManager) StatusWithBlockJobsCount(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
func (m *SGuestManager) GetGuestStatus(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
sid := params.(string)
status := m.getStatus(sid)
guest, _ := m.GetServer(sid)
body := jsonutils.NewDict()
if guest != nil {
body.Set("power_status", jsonutils.NewString(guest.GetPowerStates()))
body.Set("power_status", jsonutils.NewString(GetPowerStates(guest)))
}
if status == GUEST_RUNNING && guest.pciUninitialized {
status = compute.VM_UNSYNC
} else if status == GUEST_RUNNING {
var runCb = func() {
body := jsonutils.NewDict()
blockJobsCount := guest.BlockJobsCount()
if blockJobsCount > 0 {
status = GUEST_BLOCK_STREAM
}
body.Set("block_jobs_count", jsonutils.NewInt(int64(blockJobsCount)))
body.Set("status", jsonutils.NewString(status))
hostutils.TaskComplete(ctx, body)
}
if guest.Monitor == nil && !guest.IsStopping() {
if err := guest.StartMonitor(context.Background(), runCb, false); err != nil {
log.Errorf("guest %s failed start monitor %s", guest.GetName(), err)
body.Set("status", jsonutils.NewString(status))
hostutils.TaskComplete(ctx, body)
}
} else {
runCb()
}
return nil, nil
}
body.Set("status", jsonutils.NewString(status))
hostutils.TaskComplete(ctx, body)
return nil, nil
return guest.HandleGuestStatus(ctx, status, body)
}
func (m *SGuestManager) getStatus(sid string) string {
@@ -890,7 +864,7 @@ func (m *SGuestManager) getStatus(sid string) string {
}
}
func (m *SGuestManager) Delete(sid string) (*SKVMGuestInstance, error) {
func (m *SGuestManager) Delete(sid string) (GuestRuntimeInstance, error) {
if guest, ok := m.GetServer(sid); ok {
m.CleanServer(sid)
// 这里应该不需要append到deleted servers
@@ -908,49 +882,29 @@ func (m *SGuestManager) GuestStart(ctx context.Context, userCred mcclient.TokenC
if guest, ok := m.GetServer(sid); ok {
guestDesc := new(desc.SGuestDesc)
if err := body.Unmarshal(guestDesc, "desc"); err == nil {
if err = guest.SaveDesc(guestDesc); err != nil {
if err = SaveDesc(guest, guestDesc); err != nil {
return nil, errors.Wrap(err, "save desc")
}
}
if guest.IsStopped() {
data, err := body.Get("params")
if err != nil {
data = jsonutils.NewDict()
}
err = guest.StartGuest(ctx, userCred, data.(*jsonutils.JSONDict))
if err != nil {
return nil, err
}
res := jsonutils.NewDict()
res.Set("vnc_port", jsonutils.NewInt(0))
return res, nil
} else {
vncPort := guest.GetVncPort()
if vncPort > 0 {
res := jsonutils.NewDict()
res.Set("vnc_port", jsonutils.NewInt(int64(vncPort)))
res.Set("is_running", jsonutils.JSONTrue)
return res, nil
} else {
return nil, httperrors.NewBadRequestError("Seems started, but no VNC info")
}
}
return guest.HandleGuestStart(ctx, userCred, body)
} else {
return nil, httperrors.NewNotFoundError("Not found")
return nil, httperrors.NewNotFoundError("Not found server %s", sid)
}
}
func (m *SGuestManager) GuestStop(ctx context.Context, sid string, timeout int64) error {
if guest, ok := m.GetServer(sid); ok {
hostutils.DelayTaskWithoutReqctx(ctx, guest.ExecStopTask, timeout)
return nil
if server, ok := m.GetServer(sid); ok {
if err := server.HandleStop(ctx, timeout); err != nil {
return errors.Wrap(err, "Do stop")
}
} else {
return httperrors.NewNotFoundError("Guest %s not found", sid)
}
return nil
}
func (m *SGuestManager) GuestStartRescue(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
if guest, ok := m.GetServer(sid); ok {
if guest, ok := m.GetKVMServer(sid); ok {
// initrd and kernel should be prepared by host-deployer
if !fileutils2.Exists(guest.getRescueInitrdPath()) {
return nil, httperrors.NewInternalServerError("guest initrd not ready")
@@ -969,7 +923,7 @@ func (m *SGuestManager) GuestSync(ctx context.Context, params interface{}) (json
if !ok {
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(syncParams.Sid)
guest, _ := m.GetKVMServer(syncParams.Sid)
if syncParams.Body.Contains("desc") {
guestDesc := new(desc.SGuestDesc)
if err := syncParams.Body.Unmarshal(guestDesc, "desc"); err != nil {
@@ -987,7 +941,10 @@ func (m *SGuestManager) GuestSuspend(ctx context.Context, params interface{}) (j
if !ok {
return nil, hostutils.ParamsError
}
guest, ok := m.GetServer(sid)
guest, ok := m.GetKVMServer(sid)
if !ok {
return nil, errors.Errorf("Not found KVM server: %s", sid)
}
guest.ExecSuspendTask(ctx)
return nil, nil
}
@@ -997,17 +954,17 @@ func (m *SGuestManager) GuestIoThrottle(ctx context.Context, params interface{})
if !ok {
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(guestIoThrottle.Sid)
for i := range guest.Desc.Disks {
diskId := guest.Desc.Disks[i].DiskId
guest, _ := m.GetKVMServer(guestIoThrottle.Sid)
for i := range guest.GetDesc().Disks {
diskId := guest.GetDesc().Disks[i].DiskId
if bps, ok := guestIoThrottle.Input.Bps[diskId]; ok {
guest.Desc.Disks[i].Bps = bps
guest.GetDesc().Disks[i].Bps = bps
}
if iops, ok := guestIoThrottle.Input.IOPS[diskId]; ok {
guest.Desc.Disks[i].Iops = iops
guest.GetDesc().Disks[i].Iops = iops
}
}
if err := guest.SaveLiveDesc(guest.Desc); err != nil {
if err := SaveLiveDesc(guest, guest.GetDesc()); err != nil {
return nil, errors.Wrap(err, "guest save desc")
}
@@ -1023,7 +980,7 @@ func (m *SGuestManager) SrcPrepareMigrate(ctx context.Context, params interface{
if !ok {
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(migParams.Sid)
guest, _ := m.GetKVMServer(migParams.Sid)
disksBack, diskSnapsChain, sysDiskHasTemplate, err := guest.PrepareDisksMigrate(migParams.LiveMigrate)
if err != nil {
return nil, errors.Wrap(err, "PrepareDisksMigrate")
@@ -1047,13 +1004,13 @@ func (m *SGuestManager) SrcPrepareMigrate(ctx context.Context, params interface{
ret.Set("migrate_certs", jsonutils.Marshal(certs))
}
if migParams.LiveMigrate {
if guest.Desc.Machine == "" {
guest.Desc.Machine = guest.getMachine()
if guest.GetDesc().Machine == "" {
guest.GetDesc().Machine = guest.getMachine()
}
if err = guest.syncVirtioDiskNumQueues(); err != nil {
return nil, errors.Wrap(err, "syncVirtioDiskNumQueues")
}
ret.Set("src_desc", jsonutils.Marshal(guest.Desc))
ret.Set("src_desc", jsonutils.Marshal(guest.GetDesc()))
}
return ret, nil
}
@@ -1064,8 +1021,8 @@ func (m *SGuestManager) DestPrepareMigrate(ctx context.Context, params interface
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(migParams.Sid)
if err := guest.CreateFromDesc(migParams.Desc); err != nil {
guest, _ := m.GetKVMServer(migParams.Sid)
if err := NewGuestRuntimeManager().CreateFromDesc(guest, migParams.Desc); err != nil {
return nil, err
}
@@ -1094,7 +1051,7 @@ func (m *SGuestManager) DestPrepareMigrate(ctx context.Context, params interface
return nil, fmt.Errorf("dest prepare migrate failed %s", err)
}
}
if err := guest.SaveDesc(migParams.Desc); err != nil {
if err := SaveDesc(guest, migParams.Desc); err != nil {
log.Errorln(err)
return nil, err
}
@@ -1172,7 +1129,7 @@ func (m *SGuestManager) LiveMigrate(ctx context.Context, params interface{}) (js
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(migParams.Sid)
guest, _ := m.GetKVMServer(migParams.Sid)
task := NewGuestLiveMigrateTask(ctx, guest, migParams)
task.Start()
return nil, nil
@@ -1272,7 +1229,7 @@ func (m *SGuestManager) ReloadDiskSnapshot(
if !ok {
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(reloadParams.Sid)
guest, _ := m.GetKVMServer(reloadParams.Sid)
return guest.ExecReloadDiskTask(ctx, reloadParams.Disk)
}
@@ -1281,7 +1238,7 @@ func (m *SGuestManager) DoSnapshot(ctx context.Context, params interface{}) (jso
if !ok {
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(snapshotParams.Sid)
guest, _ := m.GetKVMServer(snapshotParams.Sid)
return guest.ExecDiskSnapshotTask(ctx, snapshotParams.UserCred, snapshotParams.Disk, snapshotParams.SnapshotId)
}
@@ -1292,7 +1249,7 @@ func (m *SGuestManager) DeleteSnapshot(ctx context.Context, params interface{})
}
if len(delParams.ConvertSnapshot) > 0 {
guest, _ := m.GetServer(delParams.Sid)
guest, _ := m.GetKVMServer(delParams.Sid)
return guest.ExecDeleteSnapshotTask(ctx, delParams.Disk, delParams.DeleteSnapshot,
delParams.ConvertSnapshot, delParams.PendingDelete)
} else {
@@ -1308,7 +1265,7 @@ func (m *SGuestManager) DoMemorySnapshot(ctx context.Context, params interface{}
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(input.Sid)
guest, _ := m.GetKVMServer(input.Sid)
return guest.ExecMemorySnapshotTask(ctx, input.GuestMemorySnapshotRequest)
}
@@ -1318,7 +1275,7 @@ func (m *SGuestManager) DoResetMemorySnapshot(ctx context.Context, params interf
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(input.Sid)
guest, _ := m.GetKVMServer(input.Sid)
return guest.ExecMemorySnapshotResetTask(ctx, input.GuestMemorySnapshotResetRequest)
}
@@ -1338,7 +1295,7 @@ func (m *SGuestManager) DoDeleteMemorySnapshot(ctx context.Context, params inter
}
func (m *SGuestManager) Resume(ctx context.Context, sid string, isLiveMigrate bool, cleanTLS bool) (jsonutils.JSONObject, error) {
guest, _ := m.GetServer(sid)
guest, _ := m.GetKVMServer(sid)
if guest.IsStopping() || guest.IsStopped() {
return nil, httperrors.NewInvalidStatusError("resume stopped server???")
}
@@ -1364,7 +1321,7 @@ func (m *SGuestManager) Resume(ctx context.Context, sid string, isLiveMigrate bo
}
func (m *SGuestManager) OnlineResizeDisk(ctx context.Context, sid string, diskId string, sizeMb int64) (jsonutils.JSONObject, error) {
guest, ok := m.GetServer(sid)
guest, ok := m.GetKVMServer(sid)
if !ok {
return nil, httperrors.NewNotFoundError("guest %s not found", sid)
}
@@ -1396,9 +1353,9 @@ func (m *SGuestManager) StartBlockReplication(ctx context.Context, params interf
if len(nbdOpts) != 3 {
return nil, fmt.Errorf("Nbd url is not vaild %s", mirrorParams.NbdServerUri)
}
guest, _ := m.GetServer(mirrorParams.Sid)
guest, _ := m.GetKVMServer(mirrorParams.Sid)
// TODO: check desc
if err := guest.SaveDesc(mirrorParams.Desc); err != nil {
if err := SaveDesc(guest, mirrorParams.Desc); err != nil {
return nil, err
}
onSucc := func() {
@@ -1429,7 +1386,7 @@ func (m *SGuestManager) CancelBlockJobs(ctx context.Context, params interface{})
hostutils.TaskFailed(ctx, fmt.Sprintf("recover: %v", r))
}
}()
guest, _ := m.GetServer(sid)
guest, _ := m.GetKVMServer(sid)
NewCancelBlockJobsTask(ctx, guest).Start()
return nil, nil
}
@@ -1450,7 +1407,7 @@ func (m *SGuestManager) CancelBlockReplication(ctx context.Context, params inter
hostutils.TaskFailed(ctx, fmt.Sprintf("recover: %v", r))
}
}()
guest, _ := m.GetServer(sid)
guest, _ := m.GetKVMServer(sid)
NewCancelBlockReplicationTask(ctx, guest).Start()
return nil, nil
}
@@ -1460,7 +1417,7 @@ func (m *SGuestManager) HotplugCpuMem(ctx context.Context, params interface{}) (
if !ok {
return nil, hostutils.ParamsError
}
guest, _ := m.GetServer(hotplugParams.Sid)
guest, _ := m.GetKVMServer(hotplugParams.Sid)
NewGuestHotplugCpuMemTask(ctx, guest, int(hotplugParams.AddCpuCount), int(hotplugParams.AddMemSize)).Start()
return nil, nil
}
@@ -1492,7 +1449,7 @@ type SStorageCloneDisk struct {
func (m *SGuestManager) StorageCloneDisk(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
input := params.(*SStorageCloneDisk)
guest, _ := m.GetServer(input.ServerId)
guest, _ := m.GetKVMServer(input.ServerId)
if guest == nil {
return nil, httperrors.NewNotFoundError("Not found guest by id %s", input.ServerId)
}
@@ -1503,7 +1460,7 @@ func (m *SGuestManager) StorageCloneDisk(ctx context.Context, params interface{}
func (m *SGuestManager) LiveChangeDisk(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
input := params.(*SStorageCloneDisk)
guest, _ := m.GetServer(input.ServerId)
guest, _ := m.GetKVMServer(input.ServerId)
if guest == nil {
return nil, httperrors.NewNotFoundError("Not found guest by id %s", input.ServerId)
}
@@ -1522,23 +1479,23 @@ func (m *SGuestManager) GetHost() hostutils.IHost {
return m.host
}
func (m *SGuestManager) RequestVerifyDirtyServer(s *SKVMGuestInstance) {
hostId := s.Desc.HostId
func (m *SGuestManager) RequestVerifyDirtyServer(s GuestRuntimeInstance) {
hostId := s.GetDesc().HostId
var body = jsonutils.NewDict()
body.Set("guest_id", jsonutils.NewString(s.Id))
body.Set("guest_id", jsonutils.NewString(s.GetInitialId()))
body.Set("host_id", jsonutils.NewString(hostId))
ret, err := modules.Servers.PerformClassAction(
hostutils.GetComputeSession(context.Background()), "dirty-server-verify", body)
if err != nil {
log.Errorf("Dirty server request start error: %s", err)
} else if jsonutils.QueryBoolean(ret, "guest_unknown_need_clean", false) {
m.Delete(s.Id)
m.Delete(s.GetInitialId())
s.CleanGuest(context.Background(), true)
}
}
func (m *SGuestManager) ResetGuestNicTrafficLimit(guestId string, input []compute.ServerNicTrafficLimit) error {
guest, ok := m.GetServer(guestId)
guest, ok := m.GetKVMServer(guestId)
if !ok {
return httperrors.NewNotFoundError("guest %s not found", guestId)
}
@@ -1551,7 +1508,7 @@ func (m *SGuestManager) ResetGuestNicTrafficLimit(guestId string, input []comput
}
}
if err := guest.SaveLiveDesc(guest.Desc); err != nil {
if err := SaveLiveDesc(guest, guest.Desc); err != nil {
return errors.Wrap(err, "guest save desc")
}
return nil
@@ -1635,7 +1592,7 @@ func (m *SGuestManager) setNicTrafficLimit(guest *SKVMGuestInstance, input compu
}
func (m *SGuestManager) SetGuestNicTrafficLimit(guestId string, input []compute.ServerNicTrafficLimit) error {
guest, ok := m.GetServer(guestId)
guest, ok := m.GetKVMServer(guestId)
if !ok {
return httperrors.NewNotFoundError("guest %s not found", guestId)
}
@@ -1649,7 +1606,7 @@ func (m *SGuestManager) SetGuestNicTrafficLimit(guestId string, input []compute.
}
}
if err := guest.SaveLiveDesc(guest.Desc); err != nil {
if err := SaveLiveDesc(guest, guest.Desc); err != nil {
return errors.Wrap(err, "guest save desc")
}

8
pkg/hostman/guestman/guesttasks.go
View File

@@ -1572,7 +1572,7 @@ func (s *SGuestResumeTask) resumeGuest() {
return
}
s.Desc.IsVolatileHost = false
s.SaveLiveDesc(s.Desc)
SaveLiveDesc(s, s.Desc)
}
s.startTime = time.Now()
@@ -1591,7 +1591,7 @@ func (s *SGuestResumeTask) SetGetTaskData(f func() (jsonutils.JSONObject, error)
func (s *SGuestResumeTask) onStartRunning() {
if s.Desc.IsVolatileHost {
s.Desc.IsVolatileHost = false
s.SaveLiveDesc(s.Desc)
SaveLiveDesc(s, s.Desc)
}
s.setCgroupPid()
@@ -2632,7 +2632,7 @@ func (task *SGuestHotplugCpuMemTask) updateGuestDesc() {
}
if task.addedCpuCount > 0 || task.addedMemSize > 0 {
task.SaveLiveDesc(task.Desc)
SaveLiveDesc(task, task.Desc)
}
if task.addedMemSize > 0 {
vncPort := task.GetVncPort()
@@ -2992,7 +2992,7 @@ func (t *SGuestLiveChangeDisk) onReopenImageSuccess(res string) {
if t.Desc.Disks[i].Index == int8(t.diskIndex) {
log.Debugf("update guest disk %s desc", t.Desc.Disks[i].DiskId)
t.Desc.Disks[i].GuestdiskJsonDesc = *t.params.TargetDiskDesc
t.SaveLiveDesc(t.Desc)
SaveLiveDesc(t, t.Desc)
break
}
}

4
pkg/hostman/guestman/libvirt.go
View File

@@ -66,8 +66,8 @@ func (m *SGuestManager) GuestCreateFromLibvirt(
}
disksPath.Set(disk.DiskId, jsonutils.NewString(iDisk.GetPath()))
}
guest, _ := m.GetServer(createConfig.Sid)
if err := guest.SaveDesc(createConfig.GuestDesc); err != nil {
guest, _ := m.GetKVMServer(createConfig.Sid)
if err := SaveDesc(guest, createConfig.GuestDesc); err != nil {
return nil, err
}

2
pkg/hostman/guestman/pci.go
View File

@@ -115,7 +115,7 @@ func (s *SKVMGuestInstance) loadGuestPciAddresses() error {
if err != nil {
return errors.Wrap(err, "load desc ensure pci address")
}
if err = s.SaveLiveDesc(s.Desc); err != nil {
if err = SaveLiveDesc(s, s.Desc); err != nil {
return errors.Wrap(err, "loadGuestPciAddresses save desc")
}
return nil

4
pkg/hostman/guestman/pci_test.go
View File

@@ -21,6 +21,7 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/log"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/hostman/guestman/arch"
"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
)
@@ -128,7 +129,8 @@ func TestSKVMGuestInstance_initGuestDesc(t *testing.T) {
s := &sKVMGuestInstance{
SKVMGuestInstance: SKVMGuestInstance{
archMan: arch.NewArch(arch.Arch_x86_64),
archMan: arch.NewArch(arch.Arch_x86_64),
sBaseGuestInstance: newBaseGuestInstance("", nil, api.HYPERVISOR_KVM),
},
//manager:
}

789
pkg/hostman/guestman/pod.go Normal file
View File

@@ -0,0 +1,789 @@
package guestman
import (
"context"
"fmt"
"io/ioutil"
"path"
"path/filepath"
"time"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/apis"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/container/device"
"yunion.io/x/onecloud/pkg/hostman/container/volume_mount"
"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
"yunion.io/x/onecloud/pkg/hostman/hostutils"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/hostman/storageman"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
computemod "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/pod"
)
type PodInstance interface {
GuestRuntimeInstance
CreateContainer(ctx context.Context, userCred mcclient.TokenCredential, id string, input *hostapi.ContainerCreateInput) (jsonutils.JSONObject, error)
StartContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerCreateInput) (jsonutils.JSONObject, error)
DeleteContainer(ctx context.Context, cred mcclient.TokenCredential, id string) (jsonutils.JSONObject, error)
SyncContainerStatus(ctx context.Context, cred mcclient.TokenCredential, ctrId string) (jsonutils.JSONObject, error)
StopContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error)
PullImage(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerPullImageInput) (jsonutils.JSONObject, error)
}
type sContainer struct {
Id string `json:"id"`
Index int `json:"index"`
CRIId string `json:"cri_id"`
}
func newContainer(id string) *sContainer {
return &sContainer{
Id: id,
}
}
type sPodGuestInstance struct {
*sBaseGuestInstance
containers map[string]*sContainer
}
func newPodGuestInstance(id string, man *SGuestManager) PodInstance {
return &sPodGuestInstance{
sBaseGuestInstance: newBaseGuestInstance(id, man, computeapi.HYPERVISOR_POD),
containers: make(map[string]*sContainer),
}
}
func (s *sPodGuestInstance) CleanGuest(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
criId := s.getCRIId()
if criId != "" {
if err := s.getCRI().RemovePod(ctx, criId); err != nil {
return nil, errors.Wrapf(err, "RemovePod with cri_id %q", criId)
}
}
return nil, DeleteHomeDir(s)
}
func (s *sPodGuestInstance) ImportServer(pendingDelete bool) {
log.Infof("======pod %s ImportServer do nothing", s.Id)
// TODO: 参考SKVMGuestInstance，可以做更多的事，比如同步状态
s.manager.SaveServer(s.Id, s)
s.manager.RemoveCandidateServer(s)
}
func (s *sPodGuestInstance) DeployFs(ctx context.Context, userCred mcclient.TokenCredential, deployInfo *deployapi.DeployInfo) (jsonutils.JSONObject, error) {
return nil, nil
}
func (s *sPodGuestInstance) IsStopped() bool {
//TODO implement me
panic("implement me")
}
func (s *sPodGuestInstance) IsSuspend() bool {
return false
}
func (s *sPodGuestInstance) getCRI() pod.CRI {
return s.manager.GetCRI()
}
func (s *sPodGuestInstance) getPod(ctx context.Context) (*runtimeapi.PodSandbox, error) {
pods, err := s.getCRI().ListPods(ctx, pod.ListPodOptions{})
if err != nil {
return nil, errors.Wrap(err, "ListPods")
}
for _, p := range pods {
if p.Metadata.Uid == s.Id {
return p, nil
}
}
return nil, errors.Wrap(httperrors.ErrNotFound, "Not found pod from containerd")
}
func (s *sPodGuestInstance) IsRunning() bool {
_, err := s.getPod(context.Background())
if err != nil {
log.Warningf("check if pod of guest %s is running", s.Id)
return false
}
return true
/*ctrs, err := s.getCRI().ListContainers(context.Background(), pod.ListContainerOptions{
PodId: s.Id,
})
if err != nil {
log.Errorf("List containers of pod %q", s.GetId())
return false
}
// TODO: container s状态应该存在每个 container 资源里面
// Pod 状态只放 guest 表
isAllRunning := true
for _, ctr := range ctrs {
if ctr.State != runtimeapi.ContainerState_CONTAINER_RUNNING {
isAllRunning = false
break
}
}
return isAllRunning*/
}
func (s *sPodGuestInstance) HandleGuestStatus(ctx context.Context, status string, body *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
body.Set("status", jsonutils.NewString(status))
hostutils.TaskComplete(ctx, body)
return nil, nil
}
func (s *sPodGuestInstance) HandleGuestStart(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
hostutils.DelayTask(ctx, func(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
resp, err := s.startPod(ctx, userCred)
if err != nil {
return nil, errors.Wrap(err, "startPod")
}
return jsonutils.Marshal(resp), nil
}, nil)
return nil, nil
}
func (s *sPodGuestInstance) HandleStop(ctx context.Context, timeout int64) error {
hostutils.DelayTask(ctx, func(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
err := s.stopPod(ctx, timeout)
if err != nil {
return nil, errors.Wrap(err, "stopPod")
}
return nil, nil
}, nil)
return nil
}
func (s *sPodGuestInstance) getCreateParams() (jsonutils.JSONObject, error) {
createParamsStr, ok := s.GetDesc().Metadata[computeapi.VM_METADATA_CREATE_PARAMS]
if !ok {
return nil, errors.Errorf("not found %s in metadata", computeapi.VM_METADATA_CREATE_PARAMS)
}
return jsonutils.ParseString(createParamsStr)
}
func (s *sPodGuestInstance) getPodCreateParams() (*computeapi.PodCreateInput, error) {
createParams, err := s.getCreateParams()
if err != nil {
return nil, errors.Wrapf(err, "getCreateParams")
}
input := new(computeapi.PodCreateInput)
if err := createParams.Unmarshal(input, "pod"); err != nil {
return nil, errors.Wrapf(err, "unmarshal to pod creation input")
}
return input, nil
}
func (s *sPodGuestInstance) getPodLogDir() string {
return filepath.Join(s.HomeDir(), "logs")
}
func (s *sPodGuestInstance) GetDisks() []*desc.SGuestDisk {
return s.GetDesc().Disks
}
func (s *sPodGuestInstance) mountPodVolumes() error {
for _, vol := range s.getContainerVolumeMounts() {
if err := volume_mount.GetDriver(vol.Type).Mount(s, vol); err != nil {
return errors.Wrapf(err, "mount volume %s", jsonutils.Marshal(vol))
}
}
return nil
}
func (s *sPodGuestInstance) umountPodVolumes() error {
for _, vol := range s.getContainerVolumeMounts() {
if err := volume_mount.GetDriver(vol.Type).Unmount(s, vol); err != nil {
return errors.Wrapf(err, "Unmount volume %s", jsonutils.Marshal(vol))
}
}
return nil
}
func (s *sPodGuestInstance) getContainerVolumeMounts() []*apis.ContainerVolumeMount {
mnts := make([]*apis.ContainerVolumeMount, 0)
for _, ctr := range s.GetDesc().Containers {
for _, vol := range ctr.Spec.VolumeMounts {
tmp := vol
mnts = append(mnts, tmp)
}
}
return mnts
}
func (s *sPodGuestInstance) GetVolumesDir() string {
return filepath.Join(s.HomeDir(), "volumes")
}
func (s *sPodGuestInstance) GetDiskMountPoint(disk storageman.IDisk) string {
return filepath.Join(s.GetVolumesDir(), disk.GetId())
}
func (s *sPodGuestInstance) startPod(ctx context.Context, userCred mcclient.TokenCredential) (*computeapi.PodStartResponse, error) {
podInput, err := s.getPodCreateParams()
if err != nil {
return nil, errors.Wrap(err, "getPodCreateParams")
}
if err := s.mountPodVolumes(); err != nil {
return nil, errors.Wrap(err, "mountPodVolumes")
}
podCfg := &runtimeapi.PodSandboxConfig{
Metadata: &runtimeapi.PodSandboxMetadata{
Name: s.GetDesc().Name,
Uid: s.GetId(),
Namespace: s.GetDesc().TenantId,
Attempt: 1,
},
Hostname: s.GetDesc().Hostname,
LogDirectory: s.getPodLogDir(),
DnsConfig: nil,
PortMappings: nil,
Labels: nil,
Annotations: nil,
Linux: &runtimeapi.LinuxPodSandboxConfig{
CgroupParent: "",
SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{
NamespaceOptions: nil,
SelinuxOptions: nil,
RunAsUser: nil,
RunAsGroup: nil,
ReadonlyRootfs: false,
SupplementalGroups: nil,
//Privileged: true,
Seccomp: nil,
Apparmor: nil,
SeccompProfilePath: "",
},
Sysctls: nil,
},
Windows: nil,
}
if len(podInput.PortMappings) != 0 {
podCfg.PortMappings = make([]*runtimeapi.PortMapping, len(podInput.PortMappings))
for idx := range podInput.PortMappings {
pm := podInput.PortMappings[idx]
runtimePm := &runtimeapi.PortMapping{
ContainerPort: pm.ContainerPort,
HostPort: pm.HostPort,
HostIp: pm.HostIp,
}
switch pm.Protocol {
case computeapi.PodPortMappingProtocolTCP:
runtimePm.Protocol = runtimeapi.Protocol_TCP
case computeapi.PodPortMappingProtocolUDP:
runtimePm.Protocol = runtimeapi.Protocol_UDP
case computeapi.PodPortMappingProtocolSCTP:
runtimePm.Protocol = runtimeapi.Protocol_SCTP
default:
return nil, errors.Errorf("invalid protocol: %q", pm.Protocol)
}
podCfg.PortMappings[idx] = runtimePm
}
}
criId, err := s.getCRI().RunPod(ctx, podCfg, "")
if err != nil {
return nil, errors.Wrap(err, "cri.RunPod")
}
if err := s.setCRIInfo(ctx, userCred, criId, podCfg); err != nil {
return nil, errors.Wrap(err, "setCRIId")
}
return &computeapi.PodStartResponse{
CRIId: criId,
IsRunning: false,
}, nil
}
func (s *sPodGuestInstance) stopPod(ctx context.Context, timeout int64) error {
if err := s.umountPodVolumes(); err != nil {
return errors.Wrapf(err, "umount pod volumes")
}
if timeout == 0 {
timeout = 15
}
ctx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
defer cancel()
return s.getCRI().StopPod(ctx, &runtimeapi.StopPodSandboxRequest{
PodSandboxId: s.getCRIId(),
})
}
func (s *sPodGuestInstance) LoadDesc() error {
if err := LoadDesc(s); err != nil {
return errors.Wrap(err, "LoadDesc")
}
if err := s.loadContainers(); err != nil {
return errors.Wrap(err, "loadContainers")
}
return nil
}
func (s *sPodGuestInstance) loadContainers() error {
s.containers = make(map[string]*sContainer)
ctrFile := s.getContainersFilePath()
if !fileutils2.Exists(ctrFile) {
log.Warningf("pod %s containers file %s doesn't exist", s.Id, ctrFile)
return nil
}
ctrStr, err := ioutil.ReadFile(ctrFile)
if err != nil {
return errors.Wrapf(err, "read %s", ctrFile)
}
obj, err := jsonutils.Parse(ctrStr)
if err != nil {
return errors.Wrapf(err, "jsonutils.Parse %s", ctrStr)
}
ctrs := make(map[string]*sContainer)
if err := obj.Unmarshal(ctrs); err != nil {
return errors.Wrapf(err, "unmarshal %s to container map", obj.String())
}
s.containers = ctrs
return nil
}
func (s *sPodGuestInstance) PostLoad(m *SGuestManager) error {
return nil
}
func (s *sPodGuestInstance) getContainerCRIId(ctrId string) (string, error) {
ctr := s.getContainer(ctrId)
if ctr == nil {
return "", errors.Wrapf(errors.ErrNotFound, "Not found container %s", ctrId)
}
return ctr.CRIId, nil
}
func (s *sPodGuestInstance) StartContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerCreateInput) (jsonutils.JSONObject, error) {
_, hasCtr := s.containers[ctrId]
needRecreate := false
if hasCtr {
status, err := s.getContainerStatus(ctx, ctrId)
if err != nil {
return nil, errors.Wrap(err, "get container status")
}
if status == computeapi.CONTAINER_STATUS_EXITED {
needRecreate = true
} else if status != computeapi.CONTAINER_STATUS_CREATED {
return nil, errors.Wrapf(err, "can't start container when status is %s", status)
}
}
if !hasCtr || needRecreate {
log.Infof("recreate container %s before starting. hasCtr: %v, needRecreate: %v", ctrId, hasCtr, needRecreate)
// delete and recreate the container before starting
if hasCtr {
if _, err := s.DeleteContainer(ctx, userCred, ctrId); err != nil {
return nil, errors.Wrap(err, "delete container before starting")
}
}
if _, err := s.CreateContainer(ctx, userCred, ctrId, input); err != nil {
return nil, errors.Wrap(err, "recreate container before starting")
}
}
criId, err := s.getContainerCRIId(ctrId)
if err != nil {
return nil, errors.Wrap(err, "get container cri id")
}
if err := s.getCRI().StartContainer(ctx, criId); err != nil {
return nil, errors.Wrap(err, "CRI.StartContainer")
}
return nil, nil
}
func (s *sPodGuestInstance) StopContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
criId, err := s.getContainerCRIId(ctrId)
if err != nil {
return nil, errors.Wrap(err, "get container cri id")
}
var timeout int64 = 0
if body.Contains("timeout") {
timeout, _ = body.Int("timeout")
}
if err := s.getCRI().StopContainer(context.Background(), criId, timeout); err != nil {
return nil, errors.Wrap(err, "CRI.StopContainer")
}
return nil, nil
}
func (s *sPodGuestInstance) getCRIId() string {
return s.GetSourceDesc().Metadata[computeapi.POD_METADATA_CRI_ID]
}
func (s *sPodGuestInstance) setCRIInfo(ctx context.Context, userCred mcclient.TokenCredential, criId string, cfg *runtimeapi.PodSandboxConfig) error {
s.Desc.Metadata[computeapi.POD_METADATA_CRI_ID] = criId
cfgStr := jsonutils.Marshal(cfg).String()
s.Desc.Metadata[computeapi.POD_METADATA_CRI_CONFIG] = cfgStr
session := auth.GetSession(ctx, userCred, options.HostOptions.Region)
if _, err := computemod.Servers.SetMetadata(session, s.GetId(), jsonutils.Marshal(map[string]string{
computeapi.POD_METADATA_CRI_ID: criId,
computeapi.POD_METADATA_CRI_CONFIG: cfgStr,
})); err != nil {
return errors.Wrapf(err, "set cri_id of pod %s", s.GetId())
}
return SaveDesc(s, s.Desc)
}
func (s *sPodGuestInstance) setContainerCRIInfo(ctx context.Context, userCred mcclient.TokenCredential, ctrId, criId string) error {
session := auth.GetSession(ctx, userCred, options.HostOptions.Region)
if _, err := computemod.Containers.SetMetadata(session, ctrId, jsonutils.Marshal(map[string]string{
computeapi.CONTAINER_METADATA_CRI_ID: criId,
})); err != nil {
return errors.Wrapf(err, "set cri_id of container %s", ctrId)
}
return nil
}
func (s *sPodGuestInstance) getPodSandboxConfig() (*runtimeapi.PodSandboxConfig, error) {
cfgStr := s.GetSourceDesc().Metadata[computeapi.POD_METADATA_CRI_CONFIG]
obj, err := jsonutils.ParseString(cfgStr)
if err != nil {
return nil, errors.Wrapf(err, "ParseString to json object: %s", cfgStr)
}
podCfg := new(runtimeapi.PodSandboxConfig)
if err := obj.Unmarshal(podCfg); err != nil {
return nil, errors.Wrap(err, "Unmarshal to PodSandboxConfig")
}
return podCfg, nil
}
func (s *sPodGuestInstance) saveContainer(id string, criId string) error {
_, ok := s.containers[id]
if ok {
return errors.Errorf("container %s already exists", criId)
}
ctr := newContainer(id)
ctr.CRIId = criId
s.containers[id] = ctr
if err := s.saveContainersFile(s.containers); err != nil {
return errors.Wrap(err, "saveContainersFile")
}
return nil
}
func (s *sPodGuestInstance) saveContainersFile(containers map[string]*sContainer) error {
content := jsonutils.Marshal(containers).String()
if err := fileutils2.FilePutContents(s.getContainersFilePath(), content, false); err != nil {
return errors.Wrapf(err, "put content %s to containers file", content)
}
return nil
}
func (s *sPodGuestInstance) getContainersFilePath() string {
return path.Join(s.HomeDir(), "containers")
}
func (s *sPodGuestInstance) getContainer(id string) *sContainer {
return s.containers[id]
}
func (s *sPodGuestInstance) CreateContainer(ctx context.Context, userCred mcclient.TokenCredential, id string, input *hostapi.ContainerCreateInput) (jsonutils.JSONObject, error) {
ctrCriId, err := s.createContainer(ctx, userCred, id, input)
if err != nil {
return nil, errors.Wrap(err, "CRI.CreateContainer")
}
if err := s.setContainerCRIInfo(ctx, userCred, id, ctrCriId); err != nil {
return nil, errors.Wrap(err, "setContainerCRIInfo")
}
return nil, nil
}
func (s *sPodGuestInstance) getContainerLogPath(ctrId string) string {
return filepath.Join(fmt.Sprintf("%s.log", ctrId))
}
func (s *sPodGuestInstance) getLxcfsMounts() []*runtimeapi.Mount {
// TODO: make lxcfs configurable or be able to auto detect
lxcfsPath := "/var/lib/lxc/lxcfs"
return []*runtimeapi.Mount{
{
ContainerPath: "/proc/uptime",
HostPath: fmt.Sprintf("%s/proc/uptime", lxcfsPath),
Readonly: true,
},
{
ContainerPath: "/proc/meminfo",
HostPath: fmt.Sprintf("%s/proc/meminfo", lxcfsPath),
Readonly: true,
},
{
ContainerPath: "/proc/stat",
HostPath: fmt.Sprintf("%s/proc/stat", lxcfsPath),
Readonly: true,
},
{
ContainerPath: "/proc/cpuinfo",
HostPath: fmt.Sprintf("%s/proc/cpuinfo", lxcfsPath),
Readonly: true,
},
{
ContainerPath: "/proc/swaps",
HostPath: fmt.Sprintf("%s/proc/swaps", lxcfsPath),
Readonly: true,
},
{
ContainerPath: "/proc/diskstats",
HostPath: fmt.Sprintf("%s/proc/diskstats", lxcfsPath),
Readonly: true,
},
}
}
func (s *sPodGuestInstance) getContainerMounts(input *hostapi.ContainerCreateInput) ([]*runtimeapi.Mount, error) {
inputMounts := input.Spec.VolumeMounts
if len(inputMounts) == 0 {
return make([]*runtimeapi.Mount, 0), nil
}
mounts := make([]*runtimeapi.Mount, len(inputMounts))
for idx, im := range inputMounts {
mnt := &runtimeapi.Mount{
ContainerPath: im.MountPath,
Readonly: im.ReadOnly,
SelinuxRelabel: im.SelinuxRelabel,
Propagation: volume_mount.GetRuntimeVolumeMountPropagation(im.Propagation),
}
hostPath, err := volume_mount.GetDriver(im.Type).GetRuntimeMountHostPath(s, im)
if err != nil {
return nil, errors.Wrapf(err, "get runtime host mount path of %s", jsonutils.Marshal(im))
}
mnt.HostPath = hostPath
mounts[idx] = mnt
}
return mounts, nil
}
func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerCreateInput) (string, error) {
log.Infof("=====container input: %s", jsonutils.Marshal(input).PrettyString())
podCfg, err := s.getPodSandboxConfig()
if err != nil {
return "", errors.Wrap(err, "getPodSandboxConfig")
}
kboxCaps := []string{
"SETPCAP",
"AUDIT_WRITE",
"SYS_CHROOT",
"CHOWN",
"DAC_OVERRIDE",
"FOWNER",
"SETGID",
"SETUID",
"SYSLOG",
"SYS_ADMIN",
"WAKE_ALARM",
"SYS_PTRACE",
"BLOCK_SUSPEND",
"MKNOD",
"KILL",
"SYS_RESOURCE",
"NET_RAW",
"NET_ADMIN",
"NET_BIND_SERVICE",
"SYS_NICE",
}
mounts, err := s.getContainerMounts(input)
if err != nil {
return "", errors.Wrap(err, "get container mounts")
}
spec := input.Spec
ctrCfg := &runtimeapi.ContainerConfig{
Metadata: &runtimeapi.ContainerMetadata{
Name: input.Name,
},
Image: &runtimeapi.ImageSpec{
Image: spec.Image,
},
Linux: &runtimeapi.LinuxContainerConfig{
//Resources: &runtimeapi.LinuxContainerResources{
// CpuPeriod: 0,
// CpuQuota: 0,
// CpuShares: 0,
// MemoryLimitInBytes: 1024 * 1024 * 4,
// OomScoreAdj: 0,
// CpusetCpus: "",
// CpusetMems: "",
// HugepageLimits: nil,
// Unified: nil,
// MemorySwapLimitInBytes: 0,
//},
SecurityContext: &runtimeapi.LinuxContainerSecurityContext{
Capabilities: &runtimeapi.Capability{
//AddCapabilities: []string{"SYS_ADMIN"},
AddCapabilities: kboxCaps,
},
//Privileged: true,
NamespaceOptions: nil,
SelinuxOptions: nil,
RunAsUser: nil,
RunAsGroup: nil,
RunAsUsername: "",
ReadonlyRootfs: false,
SupplementalGroups: nil,
NoNewPrivs: false,
MaskedPaths: nil,
ReadonlyPaths: nil,
Seccomp: nil,
Apparmor: nil,
ApparmorProfile: "",
SeccompProfilePath: "",
},
},
LogPath: s.getContainerLogPath(ctrId),
Envs: make([]*runtimeapi.KeyValue, 0),
Devices: []*runtimeapi.Device{},
Mounts: mounts,
}
if spec.EnableLxcfs {
ctrCfg.Mounts = append(ctrCfg.Mounts, s.getLxcfsMounts()...)
}
for _, env := range spec.Envs {
ctrCfg.Envs = append(ctrCfg.Envs, &runtimeapi.KeyValue{
Key: env.Key,
Value: env.Value,
})
}
if len(spec.Devices) != 0 {
for _, dev := range spec.Devices {
ctrDevs, err := device.GetDriver(dev.Type).GetRuntimeDevices(input, dev)
if err != nil {
return "", errors.Wrapf(err, "GetRuntimeDevices of %s", jsonutils.Marshal(dev))
}
ctrCfg.Devices = append(ctrCfg.Devices, ctrDevs...)
}
nvMan, err := isolated_device.GetContainerDeviceManager(isolated_device.ContainerDeviceTypeNVIDIAGPU)
if err != nil {
return "", errors.Wrapf(err, "GetContainerDeviceManager by type %q", isolated_device.ContainerDeviceTypeNVIDIAGPU)
}
if envs := nvMan.GetContainerEnvs(spec.Devices); len(envs) > 0 {
ctrCfg.Envs = append(ctrCfg.Envs, envs...)
}
}
if len(spec.Command) != 0 {
ctrCfg.Command = spec.Command
}
if len(spec.Args) != 0 {
ctrCfg.Args = spec.Args
}
criId, err := s.getCRI().CreateContainer(ctx, s.getCRIId(), podCfg, ctrCfg, false)
if err != nil {
return "", errors.Wrap(err, "cri.CreateContainer")
}
if err := s.saveContainer(ctrId, criId); err != nil {
return "", errors.Wrap(err, "saveContainer")
}
return criId, nil
}
func (s *sPodGuestInstance) DeleteContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string) (jsonutils.JSONObject, error) {
criId, err := s.getContainerCRIId(ctrId)
if err != nil && errors.Cause(err) != errors.ErrNotFound {
return nil, errors.Wrap(err, "getContainerCRIId")
}
if criId != "" {
if err := s.getCRI().RemoveContainer(ctx, criId); err != nil {
return nil, errors.Wrap(err, "cri.RemoveContainer")
}
}
// refresh local containers file
delete(s.containers, ctrId)
if err := s.saveContainersFile(s.containers); err != nil {
return nil, errors.Wrap(err, "saveContainersFile")
}
return nil, nil
}
func (s *sPodGuestInstance) getContainerStatus(ctx context.Context, ctrId string) (string, error) {
criId, err := s.getContainerCRIId(ctrId)
if err != nil {
return "", errors.Wrapf(err, "get container cri_id by %s", ctrId)
}
resp, err := s.getCRI().ContainerStatus(ctx, criId)
if err != nil {
return "", errors.Wrap(err, "cri.ContainerStatus")
}
status := computeapi.CONTAINER_STATUS_UNKNOWN
switch resp.Status.State {
case runtimeapi.ContainerState_CONTAINER_CREATED:
status = computeapi.CONTAINER_STATUS_CREATED
case runtimeapi.ContainerState_CONTAINER_RUNNING:
status = computeapi.CONTAINER_STATUS_RUNNING
case runtimeapi.ContainerState_CONTAINER_EXITED:
status = computeapi.CONTAINER_STATUS_EXITED
case runtimeapi.ContainerState_CONTAINER_UNKNOWN:
status = computeapi.CONTAINER_STATUS_UNKNOWN
}
return status, nil
}
func (s *sPodGuestInstance) SyncContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, ctrId string) (jsonutils.JSONObject, error) {
status, err := s.getContainerStatus(ctx, ctrId)
if err != nil {
return nil, errors.Wrap(err, "get container status")
}
return jsonutils.Marshal(computeapi.ContainerSyncStatusResponse{Status: status}), nil
}
func (s *sPodGuestInstance) PullImage(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerPullImageInput) (jsonutils.JSONObject, error) {
policy := input.PullPolicy
if policy == apis.ImagePullPolicyIfNotPresent || policy == "" {
// check if image is presented
img, err := s.getCRI().ImageStatus(ctx, &runtimeapi.ImageStatusRequest{
Image: &runtimeapi.ImageSpec{
Image: input.Image,
},
})
if err != nil {
return nil, errors.Wrapf(err, "cri.ImageStatus %s", input.Image)
}
if img.Image != nil {
log.Infof("image %s already exists, skipping pulling it when policy is %s", input.Image, policy)
return jsonutils.Marshal(&runtimeapi.PullImageResponse{
ImageRef: img.Image.Id,
}), nil
}
}
podCfg, err := s.getPodSandboxConfig()
if err != nil {
return nil, errors.Wrap(err, "get pod sandbox config")
}
req := &runtimeapi.PullImageRequest{
Image: &runtimeapi.ImageSpec{
Image: input.Image,
},
SandboxConfig: podCfg,
}
if input.Auth != nil {
authCfg := &runtimeapi.AuthConfig{
Username: input.Auth.Username,
Password: input.Auth.Password,
Auth: input.Auth.Auth,
ServerAddress: input.Auth.ServerAddress,
IdentityToken: input.Auth.IdentityToken,
RegistryToken: input.Auth.RegistryToken,
}
req.Auth = authCfg
}
resp, err := s.getCRI().PullImage(ctx, req)
if err != nil {
return nil, errors.Wrapf(err, "cri.PullImage %s", input.Image)
}
return jsonutils.Marshal(resp), nil
}

1
pkg/hostman/guestman/podhandlers/doc.go Normal file
View File

@@ -0,0 +1 @@
package podhandlers // import "yunion.io/x/onecloud/pkg/hostman/guestman/podhandlers"

115
pkg/hostman/guestman/podhandlers/podhandlers.go Normal file
View File

@@ -0,0 +1,115 @@
package podhandlers
import (
"context"
"fmt"
"net/http"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/hostman/guestman"
"yunion.io/x/onecloud/pkg/hostman/hostutils"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
)
const (
POD_ID = "<podId>"
CONTAINER_ID = "<containerId>"
)
type containerActionFunc func(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, containerId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error)
type containerDelayActionParams struct {
pod guestman.PodInstance
containerId string
body jsonutils.JSONObject
}
func containerActionHandler(cf containerActionFunc) appsrv.FilterHandler {
return auth.Authenticate(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
params, _, body := appsrv.FetchEnv(ctx, w, r)
podId := params[POD_ID]
ctrId := params[CONTAINER_ID]
userCred := auth.FetchUserCredential(ctx, nil)
if body == nil {
body = jsonutils.NewDict()
}
podObj, ok := guestman.GetGuestManager().GetServer(podId)
if !ok {
hostutils.Response(ctx, w, httperrors.NewNotFoundError("Not found pod %s", podId))
return
}
pod, ok := podObj.(guestman.PodInstance)
if !ok {
hostutils.Response(ctx, w, httperrors.NewBadRequestError("runtime instance is %#v", podObj))
return
}
delayParams := &containerDelayActionParams{
pod: pod,
containerId: ctrId,
body: body,
}
hostutils.DelayTask(ctx, func(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
dp := params.(*containerDelayActionParams)
return cf(ctx, userCred, dp.pod, dp.containerId, dp.body)
}, delayParams)
hostutils.ResponseOk(ctx, w)
})
}
func AddPodHandlers(prefix string, app *appsrv.Application) {
ctrHandlers := map[string]containerActionFunc{
"create": createContainer,
"start": startContainer,
"stop": stopContainer,
"delete": deleteContainer,
"sync-status": syncContainerStatus,
"pull-image": pullImage,
}
for action, f := range ctrHandlers {
app.AddHandler("POST",
fmt.Sprintf("%s/pods/%s/containers/%s/%s", prefix, POD_ID, CONTAINER_ID, action),
containerActionHandler(f))
}
}
func pullImage(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
input := new(hostapi.ContainerPullImageInput)
if err := body.Unmarshal(input); err != nil {
return nil, errors.Wrap(err, "unmarshal to ContainerPullImageInput")
}
return pod.PullImage(ctx, userCred, ctrId, input)
}
func createContainer(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, id string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
input := new(hostapi.ContainerCreateInput)
if err := body.Unmarshal(input); err != nil {
return nil, errors.Wrap(err, "unmarshal to ContainerCreateInput")
}
return pod.CreateContainer(ctx, userCred, id, input)
}
func startContainer(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, containerId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
input := new(hostapi.ContainerCreateInput)
if err := body.Unmarshal(input); err != nil {
return nil, errors.Wrap(err, "unmarshal to ContainerCreateInput")
}
return pod.StartContainer(ctx, userCred, containerId, input)
}
func stopContainer(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
return pod.StopContainer(ctx, userCred, ctrId, body)
}
func deleteContainer(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, containerId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
return pod.DeleteContainer(ctx, userCred, containerId)
}
func syncContainerStatus(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, id string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
return pod.SyncContainerStatus(ctx, userCred, id)
}

364
pkg/hostman/guestman/qemu-kvm.go
View File

@@ -66,7 +66,6 @@ import (
"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/fuseutils"
"yunion.io/x/onecloud/pkg/util/netutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
"yunion.io/x/onecloud/pkg/util/qemuimg"
"yunion.io/x/onecloud/pkg/util/regutils2"
@@ -109,18 +108,12 @@ type SKVMInstanceRuntime struct {
type SKVMGuestInstance struct {
SKVMInstanceRuntime
*sBaseGuestInstance
Id string
Monitor monitor.Monitor
manager *SGuestManager
guestAgent *qga.QemuGuestAgent
archMan arch.Arch
// runtime description, generate from source desc
Desc *desc.SGuestDesc
// source description, input from region
SourceDesc *desc.SGuestDesc
}
func NewKVMGuestInstance(id string, manager *SGuestManager) *SKVMGuestInstance {
@@ -132,9 +125,8 @@ func NewKVMGuestInstance(id string, manager *SGuestManager) *SKVMGuestInstance {
SKVMInstanceRuntime: SKVMInstanceRuntime{
blockJobTigger: make(map[string]chan struct{}),
},
Id: id,
manager: manager,
archMan: arch.NewArch(qemuArch),
sBaseGuestInstance: newBaseGuestInstance(id, manager, api.HYPERVISOR_KVM),
archMan: arch.NewArch(qemuArch),
}
}
@@ -157,7 +149,7 @@ func (s *SKVMGuestInstance) updateGuestDesc() error {
return err
}
return s.SaveLiveDesc(s.Desc)
return SaveLiveDesc(s, s.Desc)
}
func (s *SKVMGuestInstance) releaseCpuNumaPin(cpuNumaPin []*desc.SCpuNumaPin) {
@@ -205,7 +197,7 @@ func (s *SKVMGuestInstance) reallocateNumaNodes(isMigrate bool) error {
}
}
return s.SaveLiveDesc(s.Desc)
return SaveLiveDesc(s, s.Desc)
}
func (s *SKVMGuestInstance) reallocateMigrateNumaNodes() error {
@@ -250,10 +242,10 @@ func (s *SKVMGuestInstance) validateNumaAllocated(keywords string, isMigrate, is
for i := range s.Desc.CpuNumaPin {
s.Desc.CpuNumaPin[i].Vcpus = &vcpuOrder[i]
}
return s.SaveLiveDesc(s.Desc)
return SaveLiveDesc(s, s.Desc)
}
if !isHotPlug {
return s.SaveLiveDesc(s.Desc)
return SaveLiveDesc(s, s.Desc)
}
}
@@ -328,7 +320,7 @@ func (s *SKVMGuestInstance) validateNumaAllocated(keywords string, isMigrate, is
if len(s.Desc.CpuNumaPin) > 0 { // hotplug mems
s.Desc.CpuNumaPin = append(s.Desc.CpuNumaPin, cpuNumaPin...)
return s.SaveLiveDesc(s.Desc)
return SaveLiveDesc(s, s.Desc)
}
if len(vcpuOrder) > 0 {
@@ -338,7 +330,7 @@ func (s *SKVMGuestInstance) validateNumaAllocated(keywords string, isMigrate, is
}
s.Desc.CpuNumaPin = cpuNumaPin
return s.SaveLiveDesc(s.Desc)
return SaveLiveDesc(s, s.Desc)
}
func (s *SKVMGuestInstance) initLiveDescFromSourceGuest(srcDesc *desc.SGuestDesc) error {
@@ -441,7 +433,7 @@ func (s *SKVMGuestInstance) initLiveDescFromSourceGuest(srcDesc *desc.SGuestDesc
if err != nil {
return errors.Wrap(err, "initLiveDescFromSourceGuest")
}
return s.SaveLiveDesc(srcDesc)
return SaveLiveDesc(s, srcDesc)
}
func (s *SKVMGuestInstance) IsStopping() bool {
@@ -452,14 +444,6 @@ func (s *SKVMGuestInstance) IsValid() bool {
return s.Desc != nil && s.Desc.Uuid != ""
}
func (s *SKVMGuestInstance) GetId() string {
return s.Desc.Uuid
}
func (s *SKVMGuestInstance) GetName() string {
return fmt.Sprintf("%s(%s)", s.Desc.Name, s.Desc.Uuid)
}
func (s *SKVMGuestInstance) getStateFilePathRootPrefix() string {
return path.Join(s.HomeDir(), STATE_FILE_PREFIX)
}
@@ -476,22 +460,6 @@ func (s *SKVMGuestInstance) getQemuLogPath() string {
return path.Join(s.HomeDir(), "qemu.log")
}
func (s *SKVMGuestInstance) IsLoaded() bool {
return s.Desc != nil
}
func (s *SKVMGuestInstance) HomeDir() string {
return path.Join(s.manager.ServersPath, s.Id)
}
func (s *SKVMGuestInstance) PrepareDir() error {
output, err := procutils.NewCommand("mkdir", "-p", s.HomeDir()).Output()
if err != nil {
return errors.Wrapf(err, "mkdir %s failed: %s", s.HomeDir(), output)
}
return nil
}
func (s *SKVMGuestInstance) GetPidFilePath() string {
return path.Join(s.HomeDir(), "pid")
}
@@ -614,14 +582,6 @@ func (s *SKVMGuestInstance) isSelfCmdline(cmdline, uuid string) bool {
strings.Index(cmdline, uuid) >= 0
}
func (s *SKVMGuestInstance) GetDescFilePath() string {
return path.Join(s.HomeDir(), "desc")
}
func (s *SKVMGuestInstance) GetSourceDescFilePath() string {
return path.Join(s.HomeDir(), "source-desc")
}
func (s *SKVMGuestInstance) GetRescueDirPath() string {
if s.manager.host.IsAarch64() {
return path.Join("/opt/cloud/host-deployer/yunionos/aarch64")
@@ -631,53 +591,10 @@ func (s *SKVMGuestInstance) GetRescueDirPath() string {
}
func (s *SKVMGuestInstance) LoadDesc() error {
descPath := s.GetDescFilePath()
descStr, err := ioutil.ReadFile(descPath)
if err != nil {
return errors.Wrap(err, "read desc")
if err := LoadDesc(s); err != nil {
return errors.Wrap(err, "LoadDesc")
}
var (
srcDescStr []byte
srcDescPath = s.GetSourceDescFilePath()
)
if !fileutils2.Exists(srcDescPath) {
err = fileutils2.FilePutContents(srcDescPath, string(descStr), false)
if err != nil {
return errors.Wrap(err, "save source desc")
}
srcDescStr = descStr
} else {
srcDescStr, err = ioutil.ReadFile(srcDescPath)
if err != nil {
return errors.Wrap(err, "read source desc")
}
}
// parse source desc
srcGuestDesc := new(desc.SGuestDesc)
jsonSrcDesc, err := jsonutils.Parse(srcDescStr)
if err != nil {
return errors.Wrap(err, "json parse source desc")
}
err = jsonSrcDesc.Unmarshal(srcGuestDesc)
if err != nil {
return errors.Wrap(err, "unmarshal source desc")
}
s.SourceDesc = srcGuestDesc
// parse desc
guestDesc := new(desc.SGuestDesc)
jsonDesc, err := jsonutils.Parse(descStr)
if err != nil {
return errors.Wrap(err, "json parse desc")
}
err = jsonDesc.Unmarshal(guestDesc)
if err != nil {
return errors.Wrap(err, "unmarshal desc")
}
s.Desc = guestDesc
if s.IsRunning() {
if len(s.Desc.PCIControllers) > 0 {
if err := s.loadGuestPciAddresses(); err != nil {
@@ -697,6 +614,56 @@ func (s *SKVMGuestInstance) LoadDesc() error {
return nil
}
func (s *SKVMGuestInstance) PostLoad(m *SGuestManager) error {
if s.needSyncStreamDisks {
go s.sendStreamDisksComplete(context.Background())
}
return s.loadGuestCpuset(m)
}
func (s *SKVMGuestInstance) loadGuestCpuset(m *SGuestManager) error {
if s.GetPid() > 0 {
m.cpuSet.Lock.Lock()
defer m.cpuSet.Lock.Unlock()
for _, vcpuPin := range s.Desc.VcpuPin {
pcpuSet, err := cpuset.Parse(vcpuPin.Pcpus)
if err != nil {
log.Errorf("failed parse %s pcpus: %s", s.GetName(), vcpuPin.Pcpus)
continue
}
vcpuSet, err := cpuset.Parse(vcpuPin.Vcpus)
if err != nil {
log.Errorf("failed parse %s vcpus: %s", s.GetName(), vcpuPin.Vcpus)
continue
}
m.cpuSet.LoadCpus(pcpuSet.ToSlice(), vcpuSet.Size())
}
for _, numaCpuset := range s.Desc.CpuNumaPin {
pcpuSet, err := cpuset.Parse(*numaCpuset.Pcpus)
if err != nil {
log.Errorf("failed parse %s pcpus: %s", s.GetName(), *numaCpuset.Pcpus)
continue
}
vcpuCount := int(s.Desc.Cpu)
if numaCpuset.Vcpus != nil {
vcpuSet, err := cpuset.Parse(*numaCpuset.Vcpus)
if err != nil {
log.Errorf("failed parse %s vcpus: %s", s.GetName(), *numaCpuset.Vcpus)
continue
}
vcpuCount = vcpuSet.Size()
}
hostNodes := -1
if numaCpuset.HostNodes != nil {
hostNodes = int(*numaCpuset.HostNodes)
}
m.cpuSet.LoadNumaCpus(numaCpuset.SizeMB, hostNodes, pcpuSet.ToSlice(), vcpuCount)
}
}
return nil
}
func (s *SKVMGuestInstance) IsDirtyShotdown() bool {
return s.GetPid() == -2
}
@@ -911,7 +878,7 @@ func (s *SKVMGuestInstance) ImportServer(pendingDelete bool) {
if s.Desc.HostId != hostinfo.Instance().HostId {
// fix host_id
s.Desc.HostId = hostinfo.Instance().HostId
s.SaveLiveDesc(s.Desc)
SaveLiveDesc(s, s.Desc)
}
s.manager.SaveServer(s.Id, s)
@@ -1245,10 +1212,6 @@ func (s *SKVMGuestInstance) QgaPath() string {
return path.Join(s.HomeDir(), "qga.sock")
}
func (s *SKVMGuestInstance) NicTrafficRecordPath() string {
return path.Join(s.HomeDir(), "nic_traffic.json")
}
func (s *SKVMGuestInstance) InitQga() error {
guestAgent, err := qga.NewQemuGuestAgent(s.Id, s.QgaPath())
if err != nil {
@@ -1357,7 +1320,7 @@ func (s *SKVMGuestInstance) syncStatusUnsync(reason string) {
statusInput := &apis.PerformStatusInput{
Status: api.VM_UNSYNC,
Reason: reason,
PowerStates: s.GetPowerStates(),
PowerStates: GetPowerStates(s),
}
if _, err := hostutils.UpdateServerStatus(context.Background(), s.Id, statusInput); err != nil {
log.Errorf("failed update guest status %s", err)
@@ -1410,7 +1373,7 @@ func (s *SKVMGuestInstance) collectGuestDescription() error {
return errors.Wrap(err, "failed init guest devices")
}
if err := s.SaveLiveDesc(s.Desc); err != nil {
if err := SaveLiveDesc(s, s.Desc); err != nil {
return errors.Wrap(err, "failed save live desc")
}
return nil
@@ -1429,7 +1392,7 @@ func (s *SKVMGuestInstance) syncVirtioDiskNumQueues() error {
}
}
}
return s.SaveLiveDesc(s.Desc)
return SaveLiveDesc(s, s.Desc)
}
func (s *SKVMGuestInstance) getHotpluggableCPUList() ([]monitor.HotpluggableCPU, error) {
@@ -1740,7 +1703,7 @@ func (s *SKVMGuestInstance) releaseGuestCpuset() {
}
s.Desc.VcpuPin = nil
s.Desc.CpuNumaPin = nil
s.SaveLiveDesc(s.Desc)
SaveLiveDesc(s, s.Desc)
}
func (s *SKVMGuestInstance) clearCgroup(pid int) {
@@ -1870,7 +1833,7 @@ func (s *SKVMGuestInstance) SyncStatus(reason string) {
statusInput := &apis.PerformStatusInput{
Status: status,
Reason: reason,
PowerStates: s.GetPowerStates(),
PowerStates: GetPowerStates(s),
HostId: hostinfo.Instance().HostId,
}
@@ -1879,14 +1842,6 @@ func (s *SKVMGuestInstance) SyncStatus(reason string) {
}
}
func (s *SKVMGuestInstance) GetPowerStates() string {
if s.IsRunning() {
return api.VM_POWER_STATES_ON
} else {
return api.VM_POWER_STATES_OFF
}
}
func (s *SKVMGuestInstance) CheckBlockOrRunning(jobs int) {
var status = api.VM_RUNNING
@@ -1897,7 +1852,7 @@ func (s *SKVMGuestInstance) CheckBlockOrRunning(jobs int) {
var statusInput = &apis.PerformStatusInput{
Status: status,
BlockJobsCount: jobs,
PowerStates: s.GetPowerStates(),
PowerStates: GetPowerStates(s),
HostId: hostinfo.Instance().HostId,
}
_, err := hostutils.UpdateServerStatus(context.Background(), s.Id, statusInput)
@@ -1906,81 +1861,6 @@ func (s *SKVMGuestInstance) CheckBlockOrRunning(jobs int) {
}
}
func (s *SKVMGuestInstance) SaveLiveDesc(guestDesc *desc.SGuestDesc) error {
s.Desc = guestDesc
defaultGwCnt := 0
defNics := netutils2.SNicInfoList{}
// fill in ovn vpc nic bridge field
for _, nic := range s.Desc.Nics {
if nic.Bridge == "" {
nic.Bridge = getNicBridge(nic)
}
if nic.IsDefault {
defaultGwCnt++
}
defNics = defNics.Add(nic.Ip, nic.Mac, nic.Gateway)
}
// there should 1 and only 1 default gateway
if defaultGwCnt != 1 {
// fix is_default
_, defIndex := defNics.FindDefaultNicMac()
for i := range s.Desc.Nics {
if i == defIndex {
s.Desc.Nics[i].IsDefault = true
} else {
s.Desc.Nics[i].IsDefault = false
}
}
}
if err := fileutils2.FilePutContents(
s.GetDescFilePath(), jsonutils.Marshal(s.Desc).String(), false,
); err != nil {
log.Errorf("save desc failed %s", err)
return errors.Wrap(err, "save desc")
}
return nil
}
func (s *SKVMGuestInstance) SaveDesc(guestDesc *desc.SGuestDesc) error {
s.SourceDesc = guestDesc
// fill in ovn vpc nic bridge field
for _, nic := range s.SourceDesc.Nics {
if nic.Bridge == "" {
nic.Bridge = getNicBridge(nic)
}
}
if err := fileutils2.FilePutContents(
s.GetSourceDescFilePath(), jsonutils.Marshal(s.SourceDesc).String(), false,
); err != nil {
log.Errorf("save source desc failed %s", err)
return errors.Wrap(err, "source save desc")
}
if !s.IsRunning() { // if guest not running, sync live desc
liveDesc := new(desc.SGuestDesc)
if err := jsonutils.Marshal(s.SourceDesc).Unmarshal(liveDesc); err != nil {
return errors.Wrap(err, "unmarshal live desc")
}
return s.SaveLiveDesc(liveDesc)
}
return nil
}
func (s *SKVMGuestInstance) GetVpcNIC() *desc.SGuestNetwork {
for _, nic := range s.Desc.Nics {
if nic.Vpc.Provider == api.VPC_PROVIDER_OVN {
if nic.Ip != "" {
return nic
}
}
}
return nil
}
//func (s *SKVMGuestInstance) GetRescueDesc() error {
// if !s.SourceDesc.LightMode {
// return errors.Errorf("guest %s not in rescue mode", s.Id)
@@ -2038,6 +1918,32 @@ func (s *SKVMGuestInstance) prepareEncryptKeyForStart(ctx context.Context, userC
return params, nil
}
func (s *SKVMGuestInstance) HandleGuestStart(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
if s.IsStopped() {
data, err := body.Get("params")
if err != nil {
data = jsonutils.NewDict()
}
err = s.StartGuest(ctx, userCred, data.(*jsonutils.JSONDict))
if err != nil {
return nil, errors.Wrap(err, "StartGuest")
}
res := jsonutils.NewDict()
res.Set("vnc_port", jsonutils.NewInt(0))
return res, nil
} else {
vncPort := s.GetVncPort()
if vncPort > 0 {
res := jsonutils.NewDict()
res.Set("vnc_port", jsonutils.NewInt(int64(vncPort)))
res.Set("is_running", jsonutils.JSONTrue)
return res, nil
} else {
return nil, httperrors.NewBadRequestError("Seems started, but no VNC info")
}
}
}
func (s *SKVMGuestInstance) StartGuest(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict) error {
var err error
params, err = s.prepareEncryptKeyForStart(ctx, userCred, params)
@@ -2053,6 +1959,11 @@ func (s *SKVMGuestInstance) StartGuest(ctx context.Context, userCred mcclient.To
return nil
}
func (s *SKVMGuestInstance) HandleStop(ctx context.Context, timeout int64) error {
hostutils.DelayTaskWithoutReqctx(ctx, s.ExecStopTask, timeout)
return nil
}
func (s *SKVMGuestInstance) DeployFs(ctx context.Context, userCred mcclient.TokenCredential, deployInfo *deployapi.DeployInfo) (jsonutils.JSONObject, error) {
diskInfo := deployapi.DiskInfo{}
if s.isEncrypted() {
@@ -2219,11 +2130,7 @@ func (s *SKVMGuestInstance) Delete(ctx context.Context, migrated bool) error {
if err := s.delTmpDisks(ctx, migrated); err != nil {
return errors.Wrap(err, "delTmpDisks")
}
output, err := procutils.NewCommand("rm", "-rf", s.HomeDir()).Output()
if err != nil {
return errors.Wrapf(err, "rm %s failed: %s", s.HomeDir(), output)
}
return nil
return DeleteHomeDir(s)
}
func (s *SKVMGuestInstance) Stop() bool {
@@ -2323,22 +2230,6 @@ func (s *SKVMGuestInstance) ExecSuspendTask(ctx context.Context) {
NewGuestSuspendTask(s, ctx, nil).Start()
}
func (s *SKVMGuestInstance) GetNicDescMatch(mac, ip, port, bridge string) *desc.SGuestNetwork {
nics := s.Desc.Nics
for _, nic := range nics {
if bridge == "" && nic.Bridge != "" && nic.Bridge == options.HostOptions.OvnIntegrationBridge {
continue
}
if (len(mac) == 0 || netutils2.MacEqual(nic.Mac, mac)) &&
(len(ip) == 0 || nic.Ip == ip) &&
(len(port) == 0 || nic.Ifname == port) &&
(len(bridge) == 0 || nic.Bridge == bridge) {
return nic
}
}
return nil
}
func pathEqual(disk, ndisk *desc.SGuestDisk) bool {
if disk.Path != "" && ndisk.Path != "" {
return disk.Path == ndisk.Path
@@ -2603,7 +2494,7 @@ func (s *SKVMGuestInstance) SyncConfig(
var cdroms []*desc.SGuestCdrom
var floppys []*desc.SGuestFloppy
if err := s.SaveDesc(guestDesc); err != nil {
if err := SaveDesc(s, guestDesc); err != nil {
return nil, err
}
@@ -2636,7 +2527,7 @@ func (s *SKVMGuestInstance) SyncConfig(
s.Desc.SGuestRegionDesc = guestDesc.SGuestRegionDesc
s.Desc.SGuestMetaDesc = guestDesc.SGuestMetaDesc
s.SaveLiveDesc(s.Desc)
SaveLiveDesc(s, s.Desc)
if fwOnly {
res := jsonutils.NewDict()
@@ -2667,7 +2558,7 @@ func (s *SKVMGuestInstance) SyncConfig(
lenTasks := len(tasks)
var callBack = func(errs []error) {
s.SaveLiveDesc(s.Desc)
SaveLiveDesc(s, s.Desc)
if lenTasks > 0 { // devices updated, regenerate start script
vncPort := s.GetVncPort()
data := jsonutils.NewDict()
@@ -2879,13 +2770,6 @@ func (s *SKVMGuestInstance) allocGuestNumaCpuset() error {
return nil
}
func (s *SKVMGuestInstance) CreateFromDesc(desc *desc.SGuestDesc) error {
if err := s.PrepareDir(); err != nil {
return fmt.Errorf("Failed to create server dir %s", desc.Uuid)
}
return s.SaveDesc(desc)
}
func (s *SKVMGuestInstance) GetNeedMergeBackingFileDiskIndexs() []int {
res := make([]int, 0)
for _, disk := range s.Desc.Disks {
@@ -2909,7 +2793,7 @@ func (s *SKVMGuestInstance) streamDisksComplete(ctx context.Context) {
s.needSyncStreamDisks = true
}
}
if err := s.SaveLiveDesc(s.Desc); err != nil {
if err := SaveLiveDesc(s, s.Desc); err != nil {
log.Errorf("save guest desc failed %s", err)
}
if err := s.delFlatFiles(ctx); err != nil {
@@ -2932,7 +2816,7 @@ func (s *SKVMGuestInstance) sendStreamDisksComplete(ctx context.Context) {
}
s.needSyncStreamDisks = false
if err := s.SaveLiveDesc(s.Desc); err != nil {
if err := SaveLiveDesc(s, s.Desc); err != nil {
log.Errorf("save guest desc failed %s", err)
}
}
@@ -2966,7 +2850,7 @@ func (s *SKVMGuestInstance) SyncMetadata(meta *jsonutils.JSONDict) error {
func (s *SKVMGuestInstance) updateChildIndex() error {
idx := s.getQuorumChildIndex() + 1
s.Desc.Metadata[api.QUORUM_CHILD_INDEX] = strconv.Itoa(int(idx))
s.SaveLiveDesc(s.Desc)
SaveLiveDesc(s, s.Desc)
meta := jsonutils.NewDict()
meta.Set(api.QUORUM_CHILD_INDEX, jsonutils.NewInt(idx))
return s.SyncMetadata(meta)
@@ -3113,7 +2997,7 @@ func (s *SKVMGuestInstance) CleanImportMetadata() *jsonutils.JSONDict {
if meta.Length() > 0 {
// update local metadata record, after monitor started updata region record
s.SaveLiveDesc(s.Desc)
SaveLiveDesc(s, s.Desc)
return meta
}
return nil
@@ -3621,7 +3505,7 @@ func (s *SKVMGuestInstance) getVcpuThreadIdMap(guestPid int) (map[int]string, er
func (s *SKVMGuestInstance) CPUSetRemove(ctx context.Context) error {
delete(s.Desc.Metadata, api.VM_METADATA_CGROUP_CPUSET)
if err := s.SaveLiveDesc(s.Desc); err != nil {
if err := SaveLiveDesc(s, s.Desc); err != nil {
return errors.Wrap(err, "save desc after update metadata")
}
if !s.IsRunning() {
@@ -3635,3 +3519,33 @@ func (s *SKVMGuestInstance) CPUSetRemove(ctx context.Context) error {
}
return nil
}
func (s *SKVMGuestInstance) HandleGuestStatus(ctx context.Context, status string, body *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
if status == GUEST_RUNNING && s.pciUninitialized {
status = api.VM_UNSYNC
} else if status == GUEST_RUNNING {
var runCb = func() {
body := jsonutils.NewDict()
blockJobsCount := s.BlockJobsCount()
if blockJobsCount > 0 {
status = GUEST_BLOCK_STREAM
}
body.Set("block_jobs_count", jsonutils.NewInt(int64(blockJobsCount)))
body.Set("status", jsonutils.NewString(status))
hostutils.TaskComplete(ctx, body)
}
if s.Monitor == nil && !s.IsStopping() {
if err := s.StartMonitor(context.Background(), runCb, false); err != nil {
log.Errorf("guest %s failed start monitor %s", s.GetName(), err)
body.Set("status", jsonutils.NewString(status))
hostutils.TaskComplete(ctx, body)
}
} else {
runCb()
}
return nil, nil
}
body.Set("status", jsonutils.NewString(status))
hostutils.TaskComplete(ctx, body)
return nil, nil
}

307
pkg/hostman/guestman/runtime.go Normal file
View File

@@ -0,0 +1,307 @@
package guestman
import (
"context"
"fmt"
"io/ioutil"
"path"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/netutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
)
type GuestRuntimeInstance interface {
GetName() string
GetInitialId() string
GetId() string
HomeDir() string
GetDesc() *desc.SGuestDesc
SetDesc(guestDesc *desc.SGuestDesc)
GetSourceDesc() *desc.SGuestDesc
SetSourceDesc(guestDesc *desc.SGuestDesc)
GetDescFilePath() string
NicTrafficRecordPath() string
DeployFs(ctx context.Context, userCred mcclient.TokenCredential, deployInfo *deployapi.DeployInfo) (jsonutils.JSONObject, error)
GetSourceDescFilePath() string
IsRunning() bool
IsStopped() bool
IsSuspend() bool
IsLoaded() bool
GetNicDescMatch(mac, ip, port, bridge string) *desc.SGuestNetwork
CleanGuest(ctx context.Context, params interface{}) (jsonutils.JSONObject, error)
ImportServer(pendingDelete bool)
HandleGuestStatus(ctx context.Context, status string, body *jsonutils.JSONDict) (jsonutils.JSONObject, error)
HandleGuestStart(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject) (jsonutils.JSONObject, error)
HandleStop(ctx context.Context, timeout int64) error
LoadDesc() error
PostLoad(m *SGuestManager) error
}
type sBaseGuestInstance struct {
Id string
manager *SGuestManager
// runtime description, generate from source desc
Desc *desc.SGuestDesc
// source description, input from region
SourceDesc *desc.SGuestDesc
Hypervisor string
}
func newBaseGuestInstance(id string, manager *SGuestManager, hypervisor string) *sBaseGuestInstance {
return &sBaseGuestInstance{
Id: id,
manager: manager,
Hypervisor: hypervisor,
}
}
func (s *sBaseGuestInstance) GetInitialId() string {
return s.Id
}
func (s *sBaseGuestInstance) GetId() string {
return s.Desc.Uuid
}
func (s *sBaseGuestInstance) GetName() string {
return fmt.Sprintf("%s(%s)", s.Desc.Name, s.Desc.Uuid)
}
func (b *sBaseGuestInstance) HomeDir() string {
return path.Join(b.manager.ServersPath, b.Id)
}
func (s *sBaseGuestInstance) GetDescFilePath() string {
return path.Join(s.HomeDir(), "desc")
}
func (s *sBaseGuestInstance) GetDesc() *desc.SGuestDesc {
return s.Desc
}
func (s *sBaseGuestInstance) SetDesc(guestDesc *desc.SGuestDesc) {
s.Desc = guestDesc
}
func (s *sBaseGuestInstance) GetSourceDesc() *desc.SGuestDesc {
return s.SourceDesc
}
func (s *sBaseGuestInstance) SetSourceDesc(guestDesc *desc.SGuestDesc) {
s.SourceDesc = guestDesc
}
func (s *sBaseGuestInstance) GetSourceDescFilePath() string {
return path.Join(s.HomeDir(), "source-desc")
}
func (s *sBaseGuestInstance) NicTrafficRecordPath() string {
return path.Join(s.HomeDir(), "nic_traffic.json")
}
func (s *sBaseGuestInstance) IsLoaded() bool {
return s.Desc != nil
}
func (s *sBaseGuestInstance) GetNicDescMatch(mac, ip, port, bridge string) *desc.SGuestNetwork {
nics := s.Desc.Nics
for _, nic := range nics {
if bridge == "" && nic.Bridge != "" && nic.Bridge == options.HostOptions.OvnIntegrationBridge {
continue
}
if (len(mac) == 0 || netutils2.MacEqual(nic.Mac, mac)) &&
(len(ip) == 0 || nic.Ip == ip) &&
(len(port) == 0 || nic.Ifname == port) &&
(len(bridge) == 0 || nic.Bridge == bridge) {
return nic
}
}
return nil
}
type GuestRuntimeManager struct {
}
func NewGuestRuntimeManager() *GuestRuntimeManager {
return new(GuestRuntimeManager)
}
func (f *GuestRuntimeManager) NewRuntimeInstance(id string, manager *SGuestManager, hypervisor string) GuestRuntimeInstance {
switch hypervisor {
case computeapi.HYPERVISOR_KVM, "":
return NewKVMGuestInstance(id, manager)
case computeapi.HYPERVISOR_POD:
return newPodGuestInstance(id, manager)
}
log.Fatalf("Invalid hypervisor for runtime: %q", hypervisor)
return nil
}
func PrepareDir(s GuestRuntimeInstance) error {
output, err := procutils.NewCommand("mkdir", "-p", s.HomeDir()).Output()
if err != nil {
return errors.Wrapf(err, "mkdir %s failed: %s", s.HomeDir(), output)
}
return nil
}
func (f *GuestRuntimeManager) CreateFromDesc(s GuestRuntimeInstance, desc *desc.SGuestDesc) error {
if err := PrepareDir(s); err != nil {
return errors.Errorf("Failed to create server dir %s", desc.Uuid)
}
return SaveDesc(s, desc)
}
func (s *sBaseGuestInstance) GetVpcNIC() *desc.SGuestNetwork {
for _, nic := range s.Desc.Nics {
if nic.Vpc.Provider == computeapi.VPC_PROVIDER_OVN {
if nic.Ip != "" {
return nic
}
}
}
return nil
}
func LoadDesc(s GuestRuntimeInstance) error {
descPath := s.GetDescFilePath()
descStr, err := ioutil.ReadFile(descPath)
if err != nil {
return errors.Wrap(err, "read desc")
}
var (
srcDescStr []byte
srcDescPath = s.GetSourceDescFilePath()
)
if !fileutils2.Exists(srcDescPath) {
err = fileutils2.FilePutContents(srcDescPath, string(descStr), false)
if err != nil {
return errors.Wrap(err, "save source desc")
}
srcDescStr = descStr
} else {
srcDescStr, err = ioutil.ReadFile(srcDescPath)
if err != nil {
return errors.Wrap(err, "read source desc")
}
}
// parse source desc
srcGuestDesc := new(desc.SGuestDesc)
jsonSrcDesc, err := jsonutils.Parse(srcDescStr)
if err != nil {
return errors.Wrap(err, "json parse source desc")
}
err = jsonSrcDesc.Unmarshal(srcGuestDesc)
if err != nil {
return errors.Wrap(err, "unmarshal source desc")
}
s.SetSourceDesc(srcGuestDesc)
// parse desc
guestDesc := new(desc.SGuestDesc)
jsonDesc, err := jsonutils.Parse(descStr)
if err != nil {
return errors.Wrap(err, "json parse desc")
}
err = jsonDesc.Unmarshal(guestDesc)
if err != nil {
return errors.Wrap(err, "unmarshal desc")
}
s.SetDesc(guestDesc)
return nil
}
func SaveDesc(s GuestRuntimeInstance, guestDesc *desc.SGuestDesc) error {
s.SetSourceDesc(guestDesc)
// fill in ovn vpc nic bridge field
for _, nic := range s.GetSourceDesc().Nics {
if nic.Bridge == "" {
nic.Bridge = getNicBridge(nic)
}
}
if err := fileutils2.FilePutContents(
s.GetSourceDescFilePath(), jsonutils.Marshal(s.GetSourceDesc()).String(), false,
); err != nil {
log.Errorf("save source desc failed %s", err)
return errors.Wrap(err, "source save desc")
}
if !s.IsRunning() { // if guest not running, sync live desc
liveDesc := new(desc.SGuestDesc)
if err := jsonutils.Marshal(s.GetSourceDesc()).Unmarshal(liveDesc); err != nil {
return errors.Wrap(err, "unmarshal live desc")
}
return SaveLiveDesc(s, liveDesc)
}
return nil
}
func SaveLiveDesc(s GuestRuntimeInstance, guestDesc *desc.SGuestDesc) error {
s.SetDesc(guestDesc)
defaultGwCnt := 0
defNics := netutils2.SNicInfoList{}
// fill in ovn vpc nic bridge field
for _, nic := range s.GetDesc().Nics {
if nic.Bridge == "" {
nic.Bridge = getNicBridge(nic)
}
if nic.IsDefault {
defaultGwCnt++
}
defNics = defNics.Add(nic.Ip, nic.Mac, nic.Gateway)
}
// there should 1 and only 1 default gateway
if defaultGwCnt != 1 {
// fix is_default
_, defIndex := defNics.FindDefaultNicMac()
for i := range s.GetDesc().Nics {
if i == defIndex {
s.GetDesc().Nics[i].IsDefault = true
} else {
s.GetDesc().Nics[i].IsDefault = false
}
}
}
if err := fileutils2.FilePutContents(
s.GetDescFilePath(), jsonutils.Marshal(s.GetDesc()).String(), false,
); err != nil {
log.Errorf("save desc failed %s", err)
return errors.Wrap(err, "save desc")
}
return nil
}
func GetPowerStates(s GuestRuntimeInstance) string {
if s.IsRunning() {
return computeapi.VM_POWER_STATES_ON
} else {
return computeapi.VM_POWER_STATES_OFF
}
}
func DeleteHomeDir(s GuestRuntimeInstance) error {
output, err := procutils.NewCommand("rm", "-rf", s.HomeDir()).Output()
if err != nil {
return errors.Wrapf(err, "rm %s failed: %s", s.HomeDir(), output)
}
return nil
}

5
pkg/hostman/host_services.go
View File

@@ -27,12 +27,12 @@ import (
"yunion.io/x/onecloud/pkg/hostman/guestman"
"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
"yunion.io/x/onecloud/pkg/hostman/guestman/guesthandlers"
"yunion.io/x/onecloud/pkg/hostman/guestman/podhandlers"
"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
"yunion.io/x/onecloud/pkg/hostman/hosthandler"
"yunion.io/x/onecloud/pkg/hostman/hostinfo"
"yunion.io/x/onecloud/pkg/hostman/hostmetrics"
"yunion.io/x/onecloud/pkg/hostman/hostutils"
"yunion.io/x/onecloud/pkg/hostman/kubehandlers"
"yunion.io/x/onecloud/pkg/hostman/metadata"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/hostman/storageman"
@@ -149,7 +149,8 @@ func (host *SHostService) initHandlers(app *appsrv.Application) {
storagehandler.AddStorageHandler("", app)
diskhandlers.AddDiskHandler("", app)
downloader.AddDownloadHandler("", app)
kubehandlers.AddKubeAgentHandler("", app)
podhandlers.AddPodHandlers("", app)
//kubehandlers.AddKubeAgentHandler("", app)
hosthandler.AddHostHandler("", app)
app_common.ExportOptionsHandler(app, &options.HostOptions)

52
pkg/hostman/hostinfo/hostinfo.go
View File

@@ -53,9 +53,11 @@ import (
"yunion.io/x/onecloud/pkg/hostman/hostutils/hardware"
"yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
_ "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device"
"yunion.io/x/onecloud/pkg/hostman/monitor"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/hostman/storageman"
_ "yunion.io/x/onecloud/pkg/hostman/storageman/container_storage"
"yunion.io/x/onecloud/pkg/hostman/system_service"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
@@ -68,6 +70,7 @@ import (
"yunion.io/x/onecloud/pkg/util/logclient"
"yunion.io/x/onecloud/pkg/util/netutils2"
"yunion.io/x/onecloud/pkg/util/ovnutils"
"yunion.io/x/onecloud/pkg/util/pod"
"yunion.io/x/onecloud/pkg/util/procutils"
"yunion.io/x/onecloud/pkg/util/qemutils"
"yunion.io/x/onecloud/pkg/util/sysutils"
@@ -114,6 +117,12 @@ type SHostInfo struct {
SysError map[string][]api.HostError
IoScheduler string
cri pod.CRI
}
func (h *SHostInfo) GetContainerDeviceConfigurationFilePath() string {
return options.HostOptions.ContainerDeviceConfigFile
}
func (h *SHostInfo) GetIsolatedDeviceManager() isolated_device.IsolatedDeviceManager {
@@ -212,9 +221,33 @@ func (h *SHostInfo) Init() error {
}
}
if h.IsContainerHost() {
if err := h.initCRI(); err != nil {
return errors.Wrap(err, "init container runtime interface")
}
}
return nil
}
func (h *SHostInfo) initCRI() error {
cri, err := pod.NewCRI(h.GetContainerRuntimeEndpoint(), 3*time.Second)
if err != nil {
return errors.Wrapf(err, "New CRI by endpoint %q", h.GetContainerRuntimeEndpoint())
}
ver, err := cri.Version(context.Background())
if err != nil {
return errors.Wrap(err, "get runtime version")
}
log.Infof("Init container runtime: %s", ver)
h.cri = cri
return nil
}
func (h *SHostInfo) GetCRI() pod.CRI {
return h.cri
}
func (h *SHostInfo) setupOvnChassis() error {
opts := &options.HostOptions
if opts.BridgeDriver != hostbridge.DRV_OPEN_VSWITCH {
@@ -451,7 +484,7 @@ func (h *SHostInfo) prepareEnv() error {
h.EnableNativeHugepages()
hp, err := h.Mem.GetHugepages()
if err != nil {
return errors.Wrap(err, "Mem.GetHugepages")
return errors.Wrap(err, "MEM.GetHugepages")
}
for i := 0; i < len(hp); i++ {
if hp[i].SizeKb == options.HostOptions.HugepageSizeMb*1024 {
@@ -813,10 +846,10 @@ func (h *SHostInfo) detectKernelVersion() {
func (h *SHostInfo) detectSyssoftwareInfo() error {
h.detectOsDist()
h.detectKernelVersion()
if err := h.detectQemuVersion(); err != nil {
/*if err := h.detectQemuVersion(); err != nil {
log.Errorf("detect qemu version: %s", err.Error())
h.AppendHostError(fmt.Sprintf("detect qemu version: %s", err.Error()))
}
}*/
h.detectOvsVersion()
if err := h.detectOvsKOVersion(); err != nil {
log.Errorf("detect ovs kernel version: %s", err.Error())
@@ -2427,6 +2460,19 @@ func (h *SHostInfo) IsNumaAllocateEnabled() bool {
return h.enableNumaAllocate
}
func (h *SHostInfo) IsContainerdRuning() bool {
return false
}
func (h *SHostInfo) IsContainerHost() bool {
//return options.HostOptions.EnableContainerRuntime || options.HostOptions.HostType == api.HOST_TYPE_CONTAINER
return options.HostOptions.HostType == api.HOST_TYPE_CONTAINER
}
func (h *SHostInfo) GetContainerRuntimeEndpoint() string {
return options.HostOptions.ContainerRuntimeEndpoint
}
func NewHostInfo() (*SHostInfo, error) {
var res = new(SHostInfo)
res.sysinfo = &SSysInfo{}

9
pkg/hostman/hostmetrics/hostmetrics.go
View File

@@ -167,7 +167,10 @@ func (s *SGuestMonitorCollector) GetGuests() map[string]*SGuestMonitor {
gms := make(map[string]*SGuestMonitor, 0)
guestmanager := guestman.GetGuestManager()
guestmanager.Servers.Range(func(k, v interface{}) bool {
guest := v.(*guestman.SKVMGuestInstance)
guest, ok := v.(*guestman.SKVMGuestInstance)
if !ok {
return false
}
if !guest.IsValid() {
return false
}
@@ -500,7 +503,7 @@ func NewGuestMonitor(name, id string, pid int, nics []*desc.SGuestNetwork, cpuCo
}
func (m *SGuestMonitor) SetNicDown(index int) {
guest, ok := guestman.GetGuestManager().GetServer(m.Id)
guest, ok := guestman.GetGuestManager().GetKVMServer(m.Id)
if !ok {
return
}
@@ -579,7 +582,7 @@ func (m *SGuestMonitor) Netio() []*NetIOMetric {
var ifname = nic.Ifname
var nicStat *psnet.IOCountersStat
if nic.Driver == "vfio-pci" {
if guest, ok := guestman.GetGuestManager().GetServer(m.Id); ok {
if guest, ok := guestman.GetGuestManager().GetKVMServer(m.Id); ok {
dev, err := guest.GetSriovDeviceByNetworkIndex(nic.Index)
if err != nil {
log.Errorf("failed get sriov deivce by network index %s", err)

6
pkg/hostman/hostutils/hostutils.go
View File

@@ -42,6 +42,7 @@ import (
"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/pod"
)
type IHost interface {
@@ -71,6 +72,11 @@ type IHost interface {
// SyncRootPartitionUsedCapacity() error
GetKubeletConfig() kubelet.KubeletConfig
// containerd related methods
IsContainerHost() bool
GetContainerRuntimeEndpoint() string
GetCRI() pod.CRI
}
func GetComputeSession(ctx context.Context) *mcclient.ClientSession {

73
pkg/hostman/isolated_device/container_device.go Normal file
View File

@@ -0,0 +1,73 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package isolated_device
import (
"fmt"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
)
var (
containerDeviceManagers = make(map[ContainerDeviceType]IContainerDeviceManager)
)
type ContainerDeviceType string
const (
ContainerDeviceTypeCphAMDGPU ContainerDeviceType = api.CONTAINER_DEV_CPH_AMD_GPU
ContainerDeviceTypeCphASOPBinder ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER
ContainerNetintCAASIC ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC
ContainerNetintCAQuadra ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA
ContainerDeviceTypeNVIDIAGPU ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU
)
func GetContainerDeviceManager(t ContainerDeviceType) (IContainerDeviceManager, error) {
man, ok := containerDeviceManagers[t]
if !ok {
return nil, errors.Wrapf(errors.ErrNotFound, "not found container device manager by %q", t)
}
return man, nil
}
func RegisterContainerDeviceManager(man IContainerDeviceManager) {
if _, ok := containerDeviceManagers[man.GetType()]; ok {
panic(fmt.Sprintf("container device manager %s is already registered", man.GetType()))
}
containerDeviceManagers[man.GetType()] = man
}
type ContainerDevice struct {
Path string `json:"path"`
Type ContainerDeviceType `json:"type"`
VirtualNumber int `json:"virtual_number"`
}
type ContainerDeviceConfiguration struct {
Devices []*ContainerDevice `json:"devices"`
}
type IContainerDeviceManager interface {
GetType() ContainerDeviceType
NewDevices(dev *ContainerDevice) ([]IDevice, error)
NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error)
ProbeDevices() ([]IDevice, error)
GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue
}

46
pkg/hostman/isolated_device/container_device/base_dev.go Normal file
View File

@@ -0,0 +1,46 @@
package container_device
import (
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
)
type BaseDevice struct {
*isolated_device.SBaseDevice
Path string
}
func NewBaseDevice(dev *isolated_device.PCIDevice, devType isolated_device.ContainerDeviceType, devPath string) *BaseDevice {
return &BaseDevice{
SBaseDevice: isolated_device.NewBaseDevice(dev, string(devType)),
Path: devPath,
}
}
func (b BaseDevice) GetVGACmd() string {
return ""
}
func (b BaseDevice) GetCPUCmd() string {
return ""
}
func (b BaseDevice) GetQemuId() string {
return ""
}
func (c BaseDevice) CustomProbe(idx int) error {
return nil
}
func (c BaseDevice) GetDevicePath() string {
return c.Path
}
func CheckVirtualNumber(dev *isolated_device.ContainerDevice) error {
if dev.VirtualNumber <= 0 {
return errors.Errorf("virtual_number must > 0")
}
return nil
}

139
pkg/hostman/isolated_device/container_device/cph_amd_gpu.go Normal file
View File

@@ -0,0 +1,139 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package container_device
import (
"fmt"
"os"
"path"
"strings"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/pkg/errors"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
)
func init() {
isolated_device.RegisterContainerDeviceManager(newCphAMDGPUManager())
}
type cphAMDGPUManager struct{}
func newCphAMDGPUManager() *cphAMDGPUManager {
return &cphAMDGPUManager{}
}
func (m *cphAMDGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
return nil, nil
}
func (m *cphAMDGPUManager) GetType() isolated_device.ContainerDeviceType {
return isolated_device.ContainerDeviceTypeCphAMDGPU
}
func (m *cphAMDGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
if !strings.HasPrefix(dev.Path, "/dev/dri/renderD") {
return nil, errors.Errorf("device path %q doesn't start with /dev/dri/renderD", dev.Path)
}
if err := CheckVirtualNumber(dev); err != nil {
return nil, err
}
gpuDevs := make([]isolated_device.IDevice, 0)
for i := 0; i < dev.VirtualNumber; i++ {
gpuDev, err := newCphAMDGPU(dev.Path, i)
if err != nil {
return nil, errors.Wrapf(err, "new CPH AMD GPU with index %d", i)
}
gpuDevs = append(gpuDevs, gpuDev)
}
return gpuDevs, nil
}
func (m *cphAMDGPUManager) getDeviceHostPathByAddr(dev *hostapi.ContainerDevice) (string, error) {
return dev.IsolatedDevice.Path, nil
}
func (m *cphAMDGPUManager) NewContainerDevices(_ *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) {
hostPath, err := m.getDeviceHostPathByAddr(dev)
if err != nil {
return nil, errors.Wrap(err, "get device host path")
}
cDev := &runtimeapi.Device{
ContainerPath: "/dev/dri/renderD128",
HostPath: hostPath,
Permissions: "rwm",
}
return []*runtimeapi.Device{cDev}, nil
}
func (m *cphAMDGPUManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue {
return nil
}
type cphAMDGPU struct {
*BaseDevice
}
func newCphAMDGPU(devPath string, index int) (*cphAMDGPU, error) {
dir := "/dev/dri/by-path/"
entries, err := os.ReadDir(dir)
if err != nil {
return nil, errors.Wrap(err, "read dir")
}
for _, entry := range entries {
entryName := entry.Name()
fp := path.Join(dir, entryName)
linkPath, err := os.Readlink(fp)
if err != nil {
return nil, errors.Wrapf(err, "read link of %s", entry.Name())
}
linkDevPath := path.Join(dir, linkPath)
if linkDevPath == devPath {
// get pci address
if !strings.HasSuffix(entryName, "-render") {
return nil, errors.Errorf("%s isn't render device", devPath)
}
pciAddr, err := getCphAMDGPUPCIAddr(entryName)
if err != nil {
return nil, errors.Wrapf(err, "get pci address of %s", devPath)
}
pciOutput, err := isolated_device.GetPCIStrByAddr(pciAddr)
if err != nil {
return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", pciAddr)
}
dev := isolated_device.NewPCIDevice2(pciOutput[0])
dev.Addr = fmt.Sprintf("%s-%d", dev.Addr, index)
return &cphAMDGPU{
BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeCphAMDGPU, devPath),
}, nil
}
}
return nil, errors.Wrapf(errors.ErrNotFound, "%s doesn't exist in %s", devPath, dir)
}
func getCphAMDGPUPCIAddr(linkPartName string) (string, error) {
if !strings.HasPrefix(linkPartName, "pci-") {
return "", errors.Errorf("wrong link name: %s", linkPartName)
}
segs := strings.Split(linkPartName, "-")
if len(segs) != 3 {
return "", errors.Errorf("segments is not 3 after splited by -")
}
fullAddr := segs[1]
return fullAddr, nil
}

39
pkg/hostman/isolated_device/container_device/cph_amd_gpu_test.go Normal file
View File

@@ -0,0 +1,39 @@
package container_device
import "testing"
func Test_getCphAMDGPUPCIAddr(t *testing.T) {
tests := []struct {
linkPartName string
want string
wantErr bool
}{
{
linkPartName: "pci-0000:03:00.0-card",
want: "0000:03:00.0",
wantErr: false,
},
{
linkPartName: "",
want: "",
wantErr: true,
},
{
linkPartName: "pci-0000:83:00.0-render",
want: "0000:83:00.0",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.linkPartName, func(t *testing.T) {
got, err := getCphAMDGPUPCIAddr(tt.linkPartName)
if (err != nil) != tt.wantErr {
t.Errorf("getCphAMDGPUPCIAddr() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("getCphAMDGPUPCIAddr() got = %v, want %v", got, tt.want)
}
})
}
}

180
pkg/hostman/isolated_device/container_device/cph_aosp_binder.go Normal file
View File

@@ -0,0 +1,180 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package container_device
import (
"fmt"
"os"
"path/filepath"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/pkg/errors"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
)
const (
CPH_AOSP_BINDER_CONTROL_DEV_PATH = "/dev/binder-control"
CPH_AOSP_BINDER_MODEL_NAME = "CPH AOSP BINDER"
CPH_AOSP_VENDOR_ID = "0000"
CPH_AOSP_DEVICE_ID = "0000"
)
func init() {
isolated_device.RegisterContainerDeviceManager(newCphAOSPBinderManager())
}
type cphAOSPBinderManager struct {
controlDevicePath string
controlName string
}
func newCphAOSPBinderManager() *cphAOSPBinderManager {
return &cphAOSPBinderManager{}
}
func (m *cphAOSPBinderManager) GetType() isolated_device.ContainerDeviceType {
return isolated_device.ContainerDeviceTypeCphASOPBinder
}
func (m *cphAOSPBinderManager) ProbeDevices() ([]isolated_device.IDevice, error) {
return nil, nil
}
func (m *cphAOSPBinderManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
if err := CheckVirtualNumber(dev); err != nil {
return nil, err
}
if err := m.initialize(dev); err != nil {
return nil, errors.Wrap(err, "initialize")
}
devs := make([]isolated_device.IDevice, 0)
for i := 0; i < dev.VirtualNumber; i++ {
newDev, err := m.newDeviceByIndex(i)
if err != nil {
return nil, errors.Wrapf(err, "new device by index %d", i)
}
devs = append(devs, newDev)
}
return devs, nil
}
func (m *cphAOSPBinderManager) newDeviceByIndex(index int) (isolated_device.IDevice, error) {
dev, err := newCphAOSPBinder(index, m.controlDevicePath)
if err != nil {
return nil, errors.Wrap(err, "newCphAOSPBinder")
}
return dev, nil
}
func (m *cphAOSPBinderManager) initialize(dev *isolated_device.ContainerDevice) error {
ctrlPath := CPH_AOSP_BINDER_CONTROL_DEV_PATH
info, err := os.Stat(ctrlPath)
if err != nil {
return errors.Wrapf(err, "get status of path %s", ctrlPath)
}
m.controlDevicePath = ctrlPath
m.controlName = info.Name()
return nil
}
func (m *cphAOSPBinderManager) NewContainerDevices(ctrInput *hostapi.ContainerCreateInput, input *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) {
dev := input.IsolatedDevice
if err := m.ensureBinderDevice(ctrInput.Name, dev); err != nil {
return nil, errors.Wrap(err, "createBinderDevice")
}
binderFs := "/dev/binderfs"
binderDev := func(devName string) string {
return m.getBinderHostDevPath(ctrInput.Name, devName)
}
ctrDevs := []*runtimeapi.Device{
{
ContainerPath: filepath.Join(binderFs, "binder"),
HostPath: binderDev("binder"),
Permissions: "rwm",
},
{
ContainerPath: filepath.Join(binderFs, "hwbinder"),
HostPath: binderDev("hwbinder"),
Permissions: "rwm",
},
{
ContainerPath: filepath.Join(binderFs, "vndbinder"),
HostPath: binderDev("vndbinder"),
Permissions: "rwm",
},
}
return ctrDevs, nil
}
func (m *cphAOSPBinderManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue {
return nil
}
func (m *cphAOSPBinderManager) ensureBinderDeviceOldWay(dev *hostapi.ContainerIsolatedDevice) error {
if fileutils2.Exists(dev.Path) {
return nil
}
binderBin := "/opt/yunion/bin/binder_device"
baseName := filepath.Base(dev.Path)
if err := procutils.NewRemoteCommandAsFarAsPossible(binderBin, CPH_AOSP_BINDER_CONTROL_DEV_PATH, baseName).Run(); err != nil {
return errors.Wrapf(err, "call command: %s %s %s", binderBin, CPH_AOSP_BINDER_CONTROL_DEV_PATH, baseName)
}
return nil
}
func (m *cphAOSPBinderManager) getBinderHostDevPath(ctrName, devName string) string {
binderFsPath := "/dev/binderfs"
return filepath.Join(binderFsPath, ctrName, devName)
}
func (m *cphAOSPBinderManager) ensureBinderDevice(ctrName string, dev *hostapi.ContainerIsolatedDevice) error {
binderBin := "/opt/yunion/bin/binder_devices_manager"
binderDev := func(devName string) string {
return m.getBinderHostDevPath(ctrName, devName)
}
if fileutils2.Exists(binderDev("binder")) && fileutils2.Exists(binderDev("vndbinder")) && fileutils2.Exists("hwbinder") {
return nil
}
if err := procutils.NewRemoteCommandAsFarAsPossible(binderBin, ctrName).Run(); err != nil {
return errors.Wrapf(err, "call command: %s %s", binderBin, ctrName)
}
return nil
}
type cphAOSPBinder struct {
*BaseDevice
ControlPath string
}
func newCphAOSPBinder(idx int, ctrPath string) (*cphAOSPBinder, error) {
id := fmt.Sprintf("aosp_binder_%d", idx)
dev := &isolated_device.PCIDevice{
Addr: fmt.Sprintf("%d", idx),
VendorId: CPH_AOSP_VENDOR_ID,
DeviceId: CPH_AOSP_DEVICE_ID,
ModelName: CPH_AOSP_BINDER_MODEL_NAME,
}
devPath := fmt.Sprintf("/dev/%s", id)
binderDev := &cphAOSPBinder{
BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeCphASOPBinder, devPath),
ControlPath: ctrPath,
}
return binderDev, nil
}

1
pkg/hostman/isolated_device/container_device/doc.go Normal file
View File

@@ -0,0 +1 @@
package container_device // import "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device"

180
pkg/hostman/isolated_device/container_device/net_int_device.go Normal file
View File

@@ -0,0 +1,180 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package container_device
import (
"fmt"
"regexp"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
)
var (
NetintCAASICReg = regexp.MustCompile("T4\\d\\d-.*")
NetintCAQuadraReg = regexp.MustCompile("Quadra.*")
)
const (
NETINT_VENDOR_ID = "0000"
NETINT_DEVICE_ID = "0000"
)
func init() {
isolated_device.RegisterContainerDeviceManager(newNetintDeviceManager(isolated_device.ContainerNetintCAASIC, NetintCAASICReg))
isolated_device.RegisterContainerDeviceManager(newNetintDeviceManager(isolated_device.ContainerNetintCAQuadra, NetintCAQuadraReg))
}
type NetintDeviceInfo struct {
Namespace int `json:"NameSpace"`
DevicePath string `json:"DevicePath"`
Firmware string `json:"Firmware"`
Index int `json:"Index"`
ModelNumber string `json:"ModelNumber"`
ProductName string `json:"ProductName"`
SerialNumber string `json:"SerialNumber"`
UsedBytes int `json:"UsedBytes"`
MaximumLBA int `json:"MaximumLBA"`
PhysicalSize int `json:"PhysicalSize"`
SectorSize int `json:"SectorSize"`
}
type netintDeviceManager struct {
devType isolated_device.ContainerDeviceType
devRegPattern *regexp.Regexp
}
func newNetintDeviceManager(devType isolated_device.ContainerDeviceType, reg *regexp.Regexp) *netintDeviceManager {
return &netintDeviceManager{
devType: devType,
devRegPattern: reg,
}
}
func (m *netintDeviceManager) GetType() isolated_device.ContainerDeviceType {
return m.devType
}
func (m *netintDeviceManager) ProbeDevices() ([]isolated_device.IDevice, error) {
return nil, nil
}
type NVMEListResult struct {
Devices []*NetintDeviceInfo `json:"devices"`
}
func (m *netintDeviceManager) fetchNVMEDevices() ([]*NetintDeviceInfo, error) {
out, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", "nvme list -o json").Output()
if err != nil {
return nil, errors.Wrap(err, "get nvme device json output")
}
obj, err := jsonutils.Parse(out)
if err != nil {
return nil, errors.Wrapf(err, "jsonutils.Parse %s", string(out))
}
output := new(NVMEListResult)
if err := obj.Unmarshal(&output); err != nil {
return nil, errors.Wrapf(err, "Unmarshal to NetIntDeviceInfo: %s", obj.String())
}
result := make([]*NetintDeviceInfo, 0)
for _, dev := range output.Devices {
if !m.devRegPattern.MatchString(dev.ModelNumber) {
continue
}
tmpDev := dev
result = append(result, tmpDev)
}
return result, nil
}
func (m *netintDeviceManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
if err := CheckVirtualNumber(dev); err != nil {
return nil, err
}
nvmeDevs, err := m.fetchNVMEDevices()
if err != nil {
return nil, errors.Wrap(err, "fetch nvme devices")
}
result := make([]isolated_device.IDevice, 0)
for _, nvmeDev := range nvmeDevs {
for i := 0; i < dev.VirtualNumber; i++ {
newDev, err := m.newDeviceByIndex(nvmeDev, i)
if err != nil {
return nil, errors.Wrapf(err, "newDeviceByIndex %#v %d", nvmeDev, i)
}
result = append(result, newDev)
}
}
return result, nil
}
func (m *netintDeviceManager) newDeviceByIndex(dev *NetintDeviceInfo, idx int) (*netintDevice, error) {
devInfo := &isolated_device.PCIDevice{
Addr: fmt.Sprintf("%d-%d", dev.Index, idx),
VendorId: NETINT_VENDOR_ID,
DeviceId: NETINT_DEVICE_ID,
ModelName: dev.ModelNumber,
}
nvmeDev := &netintDevice{
BaseDevice: NewBaseDevice(devInfo, m.devType, dev.DevicePath),
info: dev,
}
return nvmeDev, nil
}
func (m *netintDeviceManager) NewContainerDevices(_ *hostapi.ContainerCreateInput, input *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) {
dev := input.IsolatedDevice
if !fileutils2.Exists(dev.Path) {
return nil, errors.Wrapf(httperrors.ErrNotFound, "device path %s doesn't exist", dev.Path)
}
charDevReg := regexp.MustCompile("(.*)n\\d+")
charDevPath := charDevReg.FindAllStringSubmatch(dev.Path, -1)[0][1]
ctrDevs := []*runtimeapi.Device{
{
HostPath: dev.Path,
ContainerPath: dev.Path,
Permissions: "rwm",
},
{
HostPath: charDevPath,
ContainerPath: charDevPath,
Permissions: "rwm",
},
}
return ctrDevs, nil
}
func (m *netintDeviceManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue {
return nil
}
type netintDevice struct {
*BaseDevice
info *NetintDeviceInfo
}
func (d netintDevice) GetNVMESizeMB() int {
return d.info.PhysicalSize / 1024 / 1024
}

110
pkg/hostman/isolated_device/container_device/nvidia_gpu.go Normal file
View File

@@ -0,0 +1,110 @@
package container_device
import (
"strings"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/apis/host"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
"yunion.io/x/onecloud/pkg/util/procutils"
)
func init() {
isolated_device.RegisterContainerDeviceManager(newNvidiaGPUManager())
}
type nvidiaGPUManager struct{}
func newNvidiaGPUManager() *nvidiaGPUManager {
return &nvidiaGPUManager{}
}
func (m *nvidiaGPUManager) GetType() isolated_device.ContainerDeviceType {
return isolated_device.ContainerDeviceTypeNVIDIAGPU
}
func (m *nvidiaGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
return getNvidiaGPUs()
}
func (m *nvidiaGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
return nil, nil
}
func (m *nvidiaGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) {
return nil, nil
}
func (m *nvidiaGPUManager) GetContainerEnvs(devs []*host.ContainerDevice) []*runtimeapi.KeyValue {
gpuIds := []string{}
for _, dev := range devs {
if dev.IsolatedDevice == nil {
continue
}
if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNVIDIAGPU {
continue
}
gpuIds = append(gpuIds, dev.IsolatedDevice.Path)
}
if len(gpuIds) == 0 {
return nil
}
return []*runtimeapi.KeyValue{
{
Key: "NVIDIA_VISIBLE_DEVICES",
Value: strings.Join(gpuIds, ","),
},
{
Key: "NVIDIA_DRIVER_CAPABILITIES",
Value: "all",
},
}
}
type nvidiaGPU struct {
*BaseDevice
}
func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
devs := make([]isolated_device.IDevice, 0)
// nvidia-smi --query-gpu=gpu_uuid,gpu_name,gpu_bus_id --format=csv
// uuid, name, pci.bus_id
// GPU-bc1a3bb9-55cb-8c52-c374-4f8b4f388a20, NVIDIA A800-SXM4-80GB, 00000000:10:00.0
out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id", "--format=csv").Output()
if err != nil {
return nil, errors.Wrap(err, "nvidia-smi")
}
lines := strings.Split(string(out), "\n")
for _, line := range lines {
if strings.HasPrefix(line, "uuid") {
continue
}
segs := strings.Split(line, ",")
if len(segs) != 3 {
log.Errorf("unknown nvidia-smi out line %s", line)
continue
}
gpuId, gpuName, gpuPciAddr := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2])
pciOutput, err := isolated_device.GetPCIStrByAddr(gpuPciAddr)
if err != nil {
return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", gpuPciAddr)
}
dev := isolated_device.NewPCIDevice2(pciOutput[0])
gpuDev := &nvidiaGPU{
BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNVIDIAGPU, gpuId),
}
gpuDev.SetModelName(gpuName)
devs = append(devs, gpuDev)
}
if len(devs) == 0 {
return nil, nil
}
return devs, nil
}

13
pkg/hostman/isolated_device/gpu.go
View File

@@ -127,8 +127,11 @@ func getPassthroughGPUs(filteredAddrs []string, enableWhitelist bool, whitelistM
return ret, nil, warns
}
func getGPUPCIStr() ([]string, error) {
func GetPCIStrByAddr(addr string) ([]string, error) {
cmd := "lspci -nnmm"
if addr != "" {
cmd = fmt.Sprintf("%s -s %s", cmd, addr)
}
ret, err := bashOutput(cmd)
if err != nil {
return nil, err
@@ -142,6 +145,10 @@ func getGPUPCIStr() ([]string, error) {
return lines, err
}
func getGPUPCIStr() ([]string, error) {
return GetPCIStrByAddr("")
}
type PCIDevice struct {
Addr string `json:"bus_id"`
ClassName string `json:"class_name"`
@@ -183,12 +190,12 @@ func NewPCIDevice2(line string) *PCIDevice {
}
type sGPUBaseDevice struct {
*sBaseDevice
*SBaseDevice
}
func newGPUBaseDevice(dev *PCIDevice, devType string) *sGPUBaseDevice {
return &sGPUBaseDevice{
sBaseDevice: newBaseDevice(dev, devType),
SBaseDevice: NewBaseDevice(dev, devType),
}
}

195
pkg/hostman/isolated_device/isolated_device.go
View File

@@ -51,9 +51,12 @@ type CloudDeviceInfo struct {
type IHost interface {
GetHostId() string
GetSession() *mcclient.ClientSession
IsContainerHost() bool
AppendHostError(content string)
AppendError(content, objType, id, name string)
GetContainerDeviceConfigurationFilePath() string
}
type HotPlugOption struct {
@@ -107,6 +110,7 @@ type IDevice interface {
// Get extra PCIE information
GetPCIEInfo() *api.IsolatedDevicePCIEInfo
GetDevicePath() string
}
type IsolatedDeviceManager interface {
@@ -132,7 +136,7 @@ func NewManager(host IHost) IsolatedDeviceManager {
devices: make([]IDevice, 0),
DetachedDevices: make([]*CloudDeviceInfo, 0),
}
// Do probe laster - Qiu Jian
// Do probe later - Qiu Jian
return man
}
@@ -140,6 +144,68 @@ func (man *isolatedDeviceManager) GetDevices() []IDevice {
return man.devices
}
func (man *isolatedDeviceManager) getContainerDeviceConfiguration() (*ContainerDeviceConfiguration, error) {
fp := man.host.GetContainerDeviceConfigurationFilePath()
if fp == "" {
return nil, nil
}
content, err := procutils.NewRemoteCommandAsFarAsPossible("cat", fp).Output()
if err != nil {
return nil, errors.Wrapf(err, "Read container device configuration file %s", fp)
}
obj, err := jsonutils.ParseYAML(string(content))
if err != nil {
return nil, errors.Wrapf(err, "parse YAML content: %s", content)
}
cfg := new(ContainerDeviceConfiguration)
if err := obj.Unmarshal(cfg); err != nil {
return nil, errors.Wrapf(err, "unmarshal object to ContainerDeviceConfiguration")
}
return cfg, nil
}
func (man *isolatedDeviceManager) probeContainerDevices() {
cfg, err := man.getContainerDeviceConfiguration()
panicFatal := func(err error) {
panic(err.Error())
}
if err != nil {
panicFatal(errors.Wrap(err, "get container device configuration"))
}
if cfg == nil {
return
}
for _, dev := range cfg.Devices {
devMan, err := GetContainerDeviceManager(dev.Type)
if err != nil {
panicFatal(errors.Wrapf(err, "GetContainerDeviceManager by type %q", dev.Type))
}
iDevs, err := devMan.NewDevices(dev)
if err != nil {
panicFatal(errors.Wrapf(err, "NewDevices %#v", dev))
}
man.devices = append(man.devices, iDevs...)
}
}
func (man *isolatedDeviceManager) probeContainerNvidiaGPUs() {
devman, err := GetContainerDeviceManager(ContainerDeviceTypeNVIDIAGPU)
if err != nil {
log.Errorf("no container device manager %s found", ContainerDeviceTypeNVIDIAGPU)
return
}
devs, err := devman.ProbeDevices()
if err != nil {
log.Warningf("Probe container nvidia gpu devices: %v", err)
return
} else {
for idx, dev := range devs {
man.devices = append(man.devices, dev)
log.Infof("Add Container nvidia GPU device: %d => %#v", idx, dev)
}
}
}
func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) {
if skipGPUs {
return
@@ -305,19 +371,24 @@ func (man *isolatedDeviceManager) probeNVIDIAVgpus(nvidiaVgpuPFs []string) {
func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool) {
man.devices = make([]IDevice, 0)
devModels, err := man.getCustomIsolatedDeviceModels()
if err != nil {
log.Errorf("get isolated device devModels %s", err.Error())
man.host.AppendError(fmt.Sprintf("get custom isolated device devModels %s", err.Error()), "isolated_devices", "", "")
return
if man.host.IsContainerHost() {
man.probeContainerNvidiaGPUs()
man.probeContainerDevices()
} else {
devModels, err := man.getCustomIsolatedDeviceModels()
if err != nil {
log.Errorf("get isolated device devModels %s", err.Error())
man.host.AppendError(fmt.Sprintf("get custom isolated device devModels %s", err.Error()), "isolated_devices", "", "")
return
}
man.probeUSBs(skipUSBs)
man.probeCustomPCIDevs(skipCustomDevs, devModels, GpuClassCodes)
man.probeSRIOVNics(sriovNics)
man.probeOffloadNICS(ovsOffloadNics)
man.probeAMDVgpus(amdVgpuPFs)
man.probeNVIDIAVgpus(nvidiaVgpuPFs)
man.probeGPUS(skipGPUs, amdVgpuPFs, nvidiaVgpuPFs, enableWhitelist, devModels)
}
man.probeUSBs(skipUSBs)
man.probeCustomPCIDevs(skipCustomDevs, devModels, GpuClassCodes)
man.probeSRIOVNics(sriovNics)
man.probeOffloadNICS(ovsOffloadNics)
man.probeAMDVgpus(amdVgpuPFs)
man.probeNVIDIAVgpus(nvidiaVgpuPFs)
man.probeGPUS(skipGPUs, amdVgpuPFs, nvidiaVgpuPFs, enableWhitelist, devModels)
}
type IsolatedDeviceModel struct {
@@ -421,7 +492,7 @@ func (man *isolatedDeviceManager) GetQemuParams(devAddrs []string) *QemuParams {
return getQemuParams(man, devAddrs)
}
type sBaseDevice struct {
type SBaseDevice struct {
dev *PCIDevice
cloudId string
hostId string
@@ -430,30 +501,34 @@ type sBaseDevice struct {
detectedOnHost bool
}
func newBaseDevice(dev *PCIDevice, devType string) *sBaseDevice {
return &sBaseDevice{
func NewBaseDevice(dev *PCIDevice, devType string) *SBaseDevice {
return &SBaseDevice{
dev: dev,
devType: devType,
}
}
func (dev *sBaseDevice) GetHostId() string {
return dev.hostId
}
func (dev *sBaseDevice) SetHostId(hId string) {
dev.hostId = hId
}
func (dev *sBaseDevice) String() string {
return dev.dev.String()
}
func (dev *sBaseDevice) GetWireId() string {
func (dev *SBaseDevice) GetDevicePath() string {
return ""
}
func (dev *sBaseDevice) SetDeviceInfo(info CloudDeviceInfo) {
func (dev *SBaseDevice) GetHostId() string {
return dev.hostId
}
func (dev *SBaseDevice) SetHostId(hId string) {
dev.hostId = hId
}
func (dev *SBaseDevice) String() string {
return dev.dev.String()
}
func (dev *SBaseDevice) GetWireId() string {
return ""
}
func (dev *SBaseDevice) SetDeviceInfo(info CloudDeviceInfo) {
if len(info.Id) != 0 {
dev.cloudId = info.Id
}
@@ -481,31 +556,31 @@ func SyncDeviceInfo(session *mcclient.ClientSession, hostId string, dev IDevice)
return modules.IsolatedDevices.Create(session, data)
}
func (dev *sBaseDevice) GetCloudId() string {
func (dev *SBaseDevice) GetCloudId() string {
return dev.cloudId
}
func (dev *sBaseDevice) GetVendorDeviceId() string {
func (dev *SBaseDevice) GetVendorDeviceId() string {
return dev.dev.GetVendorDeviceId()
}
func (dev *sBaseDevice) GetAddr() string {
func (dev *SBaseDevice) GetAddr() string {
return dev.dev.Addr
}
func (dev *sBaseDevice) GetDeviceType() string {
func (dev *SBaseDevice) GetDeviceType() string {
return dev.devType
}
func (dev *sBaseDevice) GetPfName() string {
func (dev *SBaseDevice) GetPfName() string {
return ""
}
func (dev *sBaseDevice) GetVirtfn() int {
func (dev *SBaseDevice) GetVirtfn() int {
return -1
}
func (dev *sBaseDevice) GetNumaNode() (int, error) {
func (dev *SBaseDevice) GetNumaNode() (int, error) {
numaNodePath := fmt.Sprintf("/sys/bus/pci/devices/0000:%s/numa_node", dev.GetAddr())
numaNode, err := fileutils2.FileGetIntContent(numaNodePath)
if err != nil {
@@ -514,27 +589,27 @@ func (dev *sBaseDevice) GetNumaNode() (int, error) {
return numaNode, nil
}
func (dev *sBaseDevice) GetOvsOffloadInterfaceName() string {
func (dev *SBaseDevice) GetOvsOffloadInterfaceName() string {
return ""
}
func (dev *sBaseDevice) IsInfinibandNic() bool {
func (dev *SBaseDevice) IsInfinibandNic() bool {
return false
}
func (dev *sBaseDevice) GetNVMESizeMB() int {
func (dev *SBaseDevice) GetNVMESizeMB() int {
return -1
}
func (dev *sBaseDevice) GetNVIDIAVgpuProfile() map[string]string {
func (dev *SBaseDevice) GetNVIDIAVgpuProfile() map[string]string {
return nil
}
func (dev *sBaseDevice) GetMdevId() string {
func (dev *SBaseDevice) GetMdevId() string {
return ""
}
func (dev *sBaseDevice) GetModelName() string {
func (dev *SBaseDevice) GetModelName() string {
if dev.dev.ModelName != "" {
return dev.dev.ModelName
} else {
@@ -542,7 +617,13 @@ func (dev *sBaseDevice) GetModelName() string {
}
}
func (dev *sBaseDevice) GetGuestId() string {
func (dev *SBaseDevice) SetModelName(modelName string) {
if dev.dev.ModelName == "" {
dev.dev.ModelName = modelName
}
}
func (dev *SBaseDevice) GetGuestId() string {
return dev.guestId
}
@@ -596,26 +677,30 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict {
if info := dev.GetPCIEInfo(); info != nil {
data["pcie_info"] = info
}
devPath := dev.GetDevicePath()
if devPath != "" {
data["device_path"] = devPath
}
return jsonutils.Marshal(data).(*jsonutils.JSONDict)
}
func (dev *sBaseDevice) GetKernelDriver() (string, error) {
func (dev *SBaseDevice) GetKernelDriver() (string, error) {
return dev.dev.getKernelDriver()
}
func (dev *sBaseDevice) getVFIODeviceCmd(addr string) string {
func (dev *SBaseDevice) getVFIODeviceCmd(addr string) string {
return fmt.Sprintf(" -device vfio-pci,host=%s", addr)
}
func (dev *sBaseDevice) GetPassthroughOptions() map[string]string {
func (dev *SBaseDevice) GetPassthroughOptions() map[string]string {
return nil
}
func (dev *sBaseDevice) GetPassthroughCmd(_ int) string {
func (dev *SBaseDevice) GetPassthroughCmd(_ int) string {
return dev.getVFIODeviceCmd(dev.GetAddr())
}
func (dev *sBaseDevice) GetIOMMUGroupRestAddrs() []string {
func (dev *SBaseDevice) GetIOMMUGroupRestAddrs() []string {
addrs := []string{}
for _, d := range dev.dev.RestIOMMUGroupDevs {
addrs = append(addrs, d.Addr)
@@ -623,7 +708,7 @@ func (dev *sBaseDevice) GetIOMMUGroupRestAddrs() []string {
return addrs
}
func (dev *sBaseDevice) GetIOMMUGroupDeviceCmd() string {
func (dev *SBaseDevice) GetIOMMUGroupDeviceCmd() string {
restAddrs := dev.GetIOMMUGroupRestAddrs()
cmds := []string{}
for _, addr := range restAddrs {
@@ -632,11 +717,11 @@ func (dev *sBaseDevice) GetIOMMUGroupDeviceCmd() string {
return strings.Join(cmds, "")
}
func (dev *sBaseDevice) DetectByAddr() error {
func (dev *SBaseDevice) DetectByAddr() error {
return nil
}
func (dev *sBaseDevice) CustomProbe(idx int) error {
func (dev *SBaseDevice) CustomProbe(idx int) error {
// check environments on first probe
if idx == 0 {
for _, driver := range []string{"vfio", "vfio_iommu_type1", "vfio-pci"} {
@@ -663,7 +748,7 @@ func (dev *sBaseDevice) CustomProbe(idx int) error {
return nil
}
func (dev *sBaseDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice, guestDesc *desc.SGuestDesc) ([]*HotPlugOption, error) {
func (dev *SBaseDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice, guestDesc *desc.SGuestDesc) ([]*HotPlugOption, error) {
ret := make([]*HotPlugOption, 0)
var masterDevOpt *HotPlugOption
@@ -703,7 +788,7 @@ func (dev *sBaseDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice
return ret, nil
}
func (dev *sBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
func (dev *SBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
if len(isolatedDev.VfioDevs) == 0 {
return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id)
}
@@ -715,7 +800,7 @@ func (dev *sBaseDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevi
}, nil
}
func (dev *sBaseDevice) GetPCIEInfo() *api.IsolatedDevicePCIEInfo {
func (dev *SBaseDevice) GetPCIEInfo() *api.IsolatedDevicePCIEInfo {
return dev.dev.PCIEInfo
}

4
pkg/hostman/isolated_device/nvidia_vgpu.go
View File

@@ -97,6 +97,10 @@ func (dev *sNVIDIAVgpuDevice) CustomProbe(idx int) error {
return nil
}
func (dev *sNVIDIAVgpuDevice) GetDevicePath() string {
return ""
}
func (dev *sNVIDIAVgpuDevice) SetDeviceInfo(info CloudDeviceInfo) {
if len(info.Id) != 0 {
dev.cloudId = info.Id

4
pkg/hostman/isolated_device/nvme.go
View File

@@ -25,7 +25,7 @@ import (
)
type sNVMEDevice struct {
*sBaseDevice
*SBaseDevice
sizeMB int
}
@@ -48,7 +48,7 @@ func (dev *sNVMEDevice) GetNVMESizeMB() int {
func newNVMEDevice(dev *PCIDevice, devType string, sizeMB int) *sNVMEDevice {
return &sNVMEDevice{
sBaseDevice: newBaseDevice(dev, devType),
SBaseDevice: NewBaseDevice(dev, devType),
sizeMB: sizeMB,
}
}

4
pkg/hostman/isolated_device/pci_device.go
View File

@@ -24,7 +24,7 @@ import (
)
type sGeneralPCIDevice struct {
*sBaseDevice
*SBaseDevice
}
func (dev *sGeneralPCIDevice) GetVGACmd() string {
@@ -41,7 +41,7 @@ func (dev *sGeneralPCIDevice) GetQemuId() string {
func newGeneralPCIDevice(dev *PCIDevice, devType string) *sGeneralPCIDevice {
return &sGeneralPCIDevice{
sBaseDevice: newBaseDevice(dev, devType),
SBaseDevice: NewBaseDevice(dev, devType),
}
}

4
pkg/hostman/isolated_device/sriov_base.go
View File

@@ -26,7 +26,7 @@ import (
)
type sSRIOVBaseDevice struct {
*sBaseDevice
*SBaseDevice
}
func ensureNumvfsEqualTotalvfs(devDir string) error {
@@ -72,7 +72,7 @@ func detectSRIOVDevice(vfBDF string) (*PCIDevice, error) {
func newSRIOVBaseDevice(dev *PCIDevice, devType string) *sSRIOVBaseDevice {
return &sSRIOVBaseDevice{
sBaseDevice: newBaseDevice(dev, devType),
SBaseDevice: NewBaseDevice(dev, devType),
}
}

4
pkg/hostman/isolated_device/usb.go
View File

@@ -30,14 +30,14 @@ import (
)
type sUSBDevice struct {
*sBaseDevice
*SBaseDevice
lsusbLine *sLsusbLine
}
// TODO: rename PCIDevice
func newUSBDevice(dev *PCIDevice, lsusbLine *sLsusbLine) *sUSBDevice {
return &sUSBDevice{
sBaseDevice: newBaseDevice(dev, api.USB_TYPE),
SBaseDevice: NewBaseDevice(dev, api.USB_TYPE),
lsusbLine: lsusbLine,
}
}

7
pkg/hostman/options/options.go
View File

@@ -55,7 +55,7 @@ type SHostOptions struct {
CommonConfigFile string `help:"common config file for container"`
LocalConfigFile string `help:"local config file" default:"/etc/yunion/host_local.conf"`
HostType string `help:"Host server type, either hypervisor or kubelet" default:"hypervisor"`
HostType string `help:"Host server type, either hypervisor or container" default:"hypervisor" choices:"hypervisor|container"`
ListenInterface string `help:"Master address of host server"`
BridgeDriver string `help:"Bridge driver, bridge or openvswitch" default:"openvswitch"`
Networks []string `help:"Network interface information"`
@@ -203,6 +203,11 @@ type SHostOptions struct {
MaxHotplugVCpuCount int `help:"maximal possible vCPU count that the platform kvm supports"`
PcieRootPortCount int `help:"pcie root port count" default:"2"`
EnableQemuDebugLog bool `help:"enable qemu debug logs" default:"false"`
// container related endpoint
// EnableContainerRuntime bool `help:"enable container runtime" default:"false"`
ContainerRuntimeEndpoint string `help:"endpoint of container runtime service" default:"unix:///var/run/onecloud/containerd/containerd.sock"`
ContainerDeviceConfigFile string `help:"container device configuration file path"`
}
var (

1
pkg/hostman/storageman/container_storage/doc.go Normal file
View File

@@ -0,0 +1 @@
package container_storage // import "yunion.io/x/onecloud/pkg/hostman/storageman/container_storage"

66
pkg/hostman/storageman/container_storage/local_raw.go Normal file
View File

@@ -0,0 +1,66 @@
package container_storage
import (
losetup "github.com/zexi/golosetup"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/isolated_device"
"yunion.io/x/onecloud/pkg/hostman/storageman"
)
func init() {
isolated_device.RegisterContainerDeviceManager(newLocalLoopDiskManager())
}
type localLoopDiskManager struct {
}
func (l localLoopDiskManager) GetType() isolated_device.ContainerDeviceType {
return api.CONTAINER_STORAGE_LOCAL_RAW
}
func (l localLoopDiskManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
return nil, errors.Errorf("%s storage doesn't support NewDevices", l.GetType())
}
func (l localLoopDiskManager) NewContainerDevices(_ *hostapi.ContainerCreateInput, input *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) {
dev := input.Disk
disk, err := storageman.GetManager().GetDiskById(dev.Id)
if err != nil {
return nil, errors.Wrapf(err, "GetDiskById %s", dev.Id)
}
format, err := disk.GetFormat()
if err != nil {
return nil, errors.Wrapf(err, "get disk %s format", dev.Id)
}
if format != "raw" {
return nil, errors.Errorf("disk %s format isn't raw", dev.Id)
}
dPath := disk.GetPath()
loDev, err := losetup.AttachDevice(dPath, false)
if err != nil {
return nil, errors.Wrapf(err, "failed to attach %s as loop device", dPath)
}
retDev := &runtimeapi.Device{
ContainerPath: input.ContainerPath,
HostPath: loDev.Name,
Permissions: "rwm",
}
return []*runtimeapi.Device{retDev}, nil
}
func (m *localLoopDiskManager) ProbeDevices() ([]isolated_device.IDevice, error) {
return nil, nil
}
func (m *localLoopDiskManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue {
return nil
}
func newLocalLoopDiskManager() *localLoopDiskManager {
return &localLoopDiskManager{}
}

13
pkg/hostman/storageman/core.go
View File

@@ -257,6 +257,19 @@ func (s *SStorageManager) GetStorageByPath(sPath string) (IStorage, error) {
return nil, errors.Wrapf(cloudprovider.ErrNotFound, sPath)
}
func (s *SStorageManager) GetDiskById(diskId string) (IDisk, error) {
for _, storage := range s.Storages {
disk, err := storage.GetDiskById(diskId)
if err != nil && errors.Cause(err) != cloudprovider.ErrNotFound {
return nil, err
}
if err == nil {
return disk, nil
}
}
return nil, errors.Wrapf(cloudprovider.ErrNotFound, diskId)
}
func (s *SStorageManager) GetDiskByPath(diskPath string) (IDisk, error) {
pos := strings.LastIndex(diskPath, "/")
sPath := diskPath[:pos]

7
pkg/hostman/storageman/disk_base.go
View File

@@ -25,6 +25,7 @@ import (
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage"
"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
@@ -75,6 +76,8 @@ type IDisk interface {
DiskBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject, error)
IsFile() bool
GetContainerStorageDriver() (container_storage.IContainerStorage, error)
}
type SBaseDisk struct {
@@ -229,3 +232,7 @@ func (d *SBaseDisk) DoDeleteSnapshot(snapshotId string) error {
func (d *SBaseDisk) GetBackupDir() string {
return ""
}
func (d *SBaseDisk) GetContainerStorageDriver() (container_storage.IContainerStorage, error) {
return nil, errors.Wrap(errors.ErrNotImplemented, "GetContainerStorageDriver")
}

18
pkg/hostman/storageman/disk_local.go
View File

@@ -31,6 +31,7 @@ import (
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage"
deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
"yunion.io/x/onecloud/pkg/hostman/hostutils"
@@ -697,3 +698,20 @@ func (d *SLocalDisk) fallocate() error {
}
return nil
}
func (d *SLocalDisk) GetContainerStorageDriver() (container_storage.IContainerStorage, error) {
format, err := d.GetFormat()
if err != nil {
return nil, errors.Wrap(err, "GetFormat")
}
// TODO: support other format
var drvType container_storage.StorageType
switch format {
case "raw":
drvType = container_storage.STORAGE_TYPE_LOCAL_RAW
default:
return nil, errors.Wrapf(errors.ErrNotImplemented, "format: %s", format)
}
drv := container_storage.GetDriver(drvType)
return drv, nil
}

37
pkg/mcclient/modules/compute/mod_containers.go Normal file
View File

@@ -0,0 +1,37 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
import (
"yunion.io/x/onecloud/pkg/mcclient/modulebase"
"yunion.io/x/onecloud/pkg/mcclient/modules"
)
type ContainerManager struct {
modulebase.ResourceManager
}
var (
Containers ContainerManager
)
func init() {
Containers = ContainerManager{
modules.NewComputeManager("container", "containers",
[]string{"ID", "Name", "Guest_ID", "Spec", "Status"},
[]string{}),
}
modules.RegisterCompute(&Containers)
}

2
pkg/mcclient/modules/compute/mod_isolatedevices.go
View File

@@ -28,7 +28,7 @@ func init() {
[]string{"ID", "Dev_type",
"Model", "Addr", "Vendor_device_id", "Mdev_id",
"Host_id", "Host",
"Guest_id", "Guest", "Guest_status", "PCIE_Info"},
"Guest_id", "Guest", "Guest_status", "Device_path", "PCIE_Info"},
[]string{})
modules.RegisterCompute(&IsolatedDevices)
}

162
pkg/mcclient/options/compute/containers.go Normal file
View File

@@ -0,0 +1,162 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
import (
"strconv"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/apis"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/mcclient/options"
)
type ContainerListOptions struct {
options.BaseListOptions
GuestId string `json:"guest_id" help:"guest(pod) id or name"`
}
func (o *ContainerListOptions) Params() (jsonutils.JSONObject, error) {
return options.ListStructToParams(o)
}
type ContainerDeleteOptions struct {
ServerIdsOptions
}
type ContainerCreateOptions struct {
PODID string `help:"Name or id of server pod" json:"-"`
NAME string `help:"Name of container" json:"-"`
IMAGE string `help:"Image of container" json:"image"`
Command []string `help:"Command to execute (i.e., entrypoint for docker)" json:"command"`
Args []string `help:"Args for the Command (i.e. command for docker)" json:"args"`
WorkingDir string `help:"Current working directory of the command" json:"working_dir"`
Env []string `help:"List of environment variable to set in the container and the format is: <key>=<value>"`
VolumeMount []string `help:"Volume mount of the container and the format is: name=<val>,mount=<container_path>,readonly=<true_or_false>,disk_index=<disk_number>,disk_id=<disk_id>"`
}
func (o *ContainerCreateOptions) Params() (jsonutils.JSONObject, error) {
req := computeapi.ContainerCreateInput{
GuestId: o.PODID,
Spec: computeapi.ContainerSpec{
ContainerSpec: apis.ContainerSpec{
Image: o.IMAGE,
Command: o.Command,
Args: o.Args,
WorkingDir: o.WorkingDir,
Envs: make([]*apis.ContainerKeyValue, 0),
VolumeMounts: make([]*apis.ContainerVolumeMount, 0),
},
},
}
req.Name = o.NAME
for _, env := range o.Env {
e, err := parseContainerEnv(env)
if err != nil {
return nil, errors.Wrapf(err, "parseContainerEnv %s", env)
}
req.Spec.Envs = append(req.Spec.Envs, e)
}
for _, vmStr := range o.VolumeMount {
vm, err := parseContainerVolumeMount(vmStr)
if err != nil {
return nil, errors.Wrapf(err, "parseContainerVolumeMount %s", vmStr)
}
req.Spec.VolumeMounts = append(req.Spec.VolumeMounts, vm)
}
return jsonutils.Marshal(req), nil
}
func parseContainerEnv(env string) (*apis.ContainerKeyValue, error) {
kv := strings.Split(env, "=")
if len(kv) != 2 {
return nil, errors.Errorf("invalid env: %q", env)
}
return &apis.ContainerKeyValue{
Key: kv[0],
Value: kv[1],
}, nil
}
func parseContainerVolumeMount(vmStr string) (*apis.ContainerVolumeMount, error) {
vm := &apis.ContainerVolumeMount{}
for _, seg := range strings.Split(vmStr, ",") {
info := strings.Split(seg, "=")
if len(info) != 2 {
return nil, errors.Errorf("invalid option %s", seg)
}
key := info[0]
val := info[1]
switch key {
case "read_only", "ro", "readonly":
if strings.ToLower(val) == "true" {
vm.ReadOnly = true
}
case "mount_path":
vm.MountPath = val
case "host_path":
if vm.HostPath == nil {
vm.HostPath = &apis.ContainerVolumeMountHostPath{}
}
vm.Type = apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH
vm.HostPath.Path = val
case "disk_index":
vm.Type = apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK
if vm.Disk == nil {
vm.Disk = &apis.ContainerVolumeMountDisk{}
}
index, err := strconv.Atoi(val)
if err != nil {
return nil, errors.Wrapf(err, "wrong disk_index %s", val)
}
vm.Disk.Index = &index
case "disk_id":
vm.Type = apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK
if vm.Disk == nil {
vm.Disk = &apis.ContainerVolumeMountDisk{}
}
vm.Disk.Id = val
case "disk_subdir", "disk_sub_dir", "disk_sub_directory":
vm.Type = apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK
if vm.Disk == nil {
vm.Disk = &apis.ContainerVolumeMountDisk{}
}
vm.Disk.SubDirectory = val
case "disk_storage_size_file", "disk_ssf":
vm.Type = apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK
if vm.Disk == nil {
vm.Disk = &apis.ContainerVolumeMountDisk{}
}
vm.Disk.StorageSizeFile = val
}
}
return vm, nil
}
type ContainerStopOptions struct {
ServerIdsOptions
Timeout int `help:"Stopping timeout" json:"timeout"`
}
func (o *ContainerStopOptions) Params() (jsonutils.JSONObject, error) {
return jsonutils.Marshal(o), nil
}
type ContainerStartOptions struct {
ServerIdsOptions
}

110
pkg/mcclient/options/compute/containers_test.go Normal file
View File

@@ -0,0 +1,110 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
import (
"reflect"
"testing"
"yunion.io/x/onecloud/pkg/apis"
)
func Test_parseContainerVolumeMount(t *testing.T) {
index0 := 0
tests := []struct {
args string
want *apis.ContainerVolumeMount
wantErr bool
}{
{
args: "readonly=true,mount_path=/data,disk_index=0",
want: &apis.ContainerVolumeMount{
ReadOnly: true,
MountPath: "/data",
Disk: &apis.ContainerVolumeMountDisk{Index: &index0},
Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
},
},
{
args: "readonly=true,mount_path=/data,disk_index=0,disk_subdir=data",
want: &apis.ContainerVolumeMount{
ReadOnly: true,
MountPath: "/data",
Disk: &apis.ContainerVolumeMountDisk{Index: &index0, SubDirectory: "data"},
Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
},
},
{
args: "readonly=true,mount_path=/storage_size,disk_index=0,disk_ssf=storage_size",
want: &apis.ContainerVolumeMount{
ReadOnly: true,
MountPath: "/storage_size",
Disk: &apis.ContainerVolumeMountDisk{
Index: &index0,
StorageSizeFile: "storage_size",
},
Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
},
},
{
args: "disk_id=abc,mount_path=/data",
wantErr: false,
want: &apis.ContainerVolumeMount{
Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
Disk: &apis.ContainerVolumeMountDisk{
Id: "abc",
},
MountPath: "/data",
},
},
{
args: "host_path=/hostpath/abc,mount_path=/data",
want: &apis.ContainerVolumeMount{
Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH,
HostPath: &apis.ContainerVolumeMountHostPath{Path: "/hostpath/abc"},
MountPath: "/data",
},
},
{
args: "read_only=True,mount_path=/test",
want: &apis.ContainerVolumeMount{
ReadOnly: true,
MountPath: "/test",
},
},
{
args: "vm1,read_only=True,mount_path=/test",
want: nil,
wantErr: true,
},
{
args: "read_only=True,mount_path=/test,disk_index=one",
want: nil,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.args, func(t *testing.T) {
got, err := parseContainerVolumeMount(tt.args)
if (err != nil) != tt.wantErr {
t.Errorf("parseContainerVolumeMount() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseContainerVolumeMount() got = %v, want %v", got, tt.want)
}
})
}
}

200
pkg/mcclient/options/compute/server_pod.go Normal file
View File

@@ -0,0 +1,200 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
import (
"fmt"
"strconv"
"strings"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/fileutils"
"yunion.io/x/pkg/util/regutils"
"yunion.io/x/onecloud/pkg/apis"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/mcclient/options"
)
type PodCreateOptions struct {
NAME string `help:"Name of server pod" json:"-"`
IMAGE string `help:"Image of container" json:"-"`
MEM string `help:"Memory size MB" metavar:"MEM" json:"-"`
VcpuCount int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"<SERVER_CPU_COUNT>" json:"vcpu_count" token:"ncpu"`
AllowDelete *bool `help:"Unlock server to allow deleting" json:"-"`
PortMapping []string `help:"Port mapping of the pod and the format is: <host_port>:<container_port>/<tcp|udp>" short-token:"p"`
Arch string `help:"image arch" choices:"aarch64|x86_64"`
Command []string `help:"Command to execute (i.e., entrypoint for docker)" json:"command"`
Args []string `help:"Args for the Command (i.e. command for docker)" json:"args"`
WorkingDir string `help:"Current working directory of the command" json:"working_dir"`
Volume []string `help:"Volume specification: name=<name>,disk_index=<index>, e.g.: name=disk0,disk_index=0"`
Device []string `help:"Host device: <host_path>:<container_path>:<permissions>, e.g.: /dev/snd:/dev/snd:rwm"`
Env []string `help:"List of environment variable to set in the container and format is: <key>=<value>"`
EnableLxcfs bool `help:"Enable lxcfs"`
VolumeMount []string `help:"Volume mount of the container and the format is: name=<val>,mount=<container_path>,readonly=<true_or_false>"`
ServerCreateCommonConfig
}
func parsePodPortMapping(input string) (*computeapi.PodPortMapping, error) {
segs := strings.Split(input, ":")
if len(segs) != 2 {
return nil, errors.Errorf("wrong format: %s", input)
}
hostPortStr := segs[0]
hostPort, err := strconv.Atoi(hostPortStr)
if err != nil {
return nil, errors.Wrapf(err, "host_port %s isn't integer", hostPortStr)
}
ctrPortPart := segs[1]
ctrPortSegs := strings.Split(ctrPortPart, "/")
if len(ctrPortSegs) > 2 {
return nil, errors.Wrapf(err, "wrong format: %s", ctrPortPart)
}
ctrPortStr := ctrPortSegs[0]
ctrPort, err := strconv.Atoi(ctrPortStr)
if err != nil {
return nil, errors.Wrapf(err, "container_port %s isn't integer", ctrPortStr)
}
var protocol computeapi.PodPortMappingProtocol = computeapi.PodPortMappingProtocolTCP
if len(ctrPortSegs) == 2 {
switch ctrPortSegs[1] {
case "tcp":
protocol = computeapi.PodPortMappingProtocolTCP
case "udp":
protocol = computeapi.PodPortMappingProtocolUDP
case "sctp":
protocol = computeapi.PodPortMappingProtocolSCTP
default:
return nil, errors.Wrapf(err, "wrong protocol: %s", ctrPortSegs[1])
}
}
return &computeapi.PodPortMapping{
Protocol: protocol,
ContainerPort: int32(ctrPort),
HostPort: int32(hostPort),
}, nil
}
func parseContainerDevice(dev string) (*computeapi.ContainerDevice, error) {
segs := strings.Split(dev, ":")
if len(segs) != 3 {
return nil, errors.Errorf("wrong format: %s", dev)
}
return &computeapi.ContainerDevice{
Type: apis.CONTAINER_DEVICE_TYPE_HOST,
Host: &computeapi.ContainerHostDevice{
ContainerPath: segs[1],
HostPath: segs[0],
Permissions: segs[2],
},
}, nil
}
func (o *PodCreateOptions) Params() (*computeapi.ServerCreateInput, error) {
config, err := o.ServerCreateCommonConfig.Data()
if err != nil {
return nil, errors.Wrapf(err, "get ServerCreateCommonConfig.Data")
}
config.Hypervisor = computeapi.HYPERVISOR_POD
portMappings := make([]*computeapi.PodPortMapping, 0)
if len(o.PortMapping) != 0 {
for _, input := range o.PortMapping {
pm, err := parsePodPortMapping(input)
if err != nil {
return nil, errors.Wrapf(err, "parse port mapping: %s", input)
}
portMappings = append(portMappings, pm)
}
}
devs := make([]*computeapi.ContainerDevice, len(o.Device))
for idx, devStr := range o.Device {
dev, err := parseContainerDevice(devStr)
if err != nil {
return nil, errors.Wrap(err, "parseContainerDevice")
}
devs[idx] = dev
}
envs := make([]*apis.ContainerKeyValue, 0)
for _, env := range o.Env {
e, err := parseContainerEnv(env)
if err != nil {
return nil, errors.Wrapf(err, "parseContainerEnv %s", env)
}
envs = append(envs, e)
}
vms := make([]*apis.ContainerVolumeMount, 0)
for _, vmStr := range o.VolumeMount {
vm, err := parseContainerVolumeMount(vmStr)
if err != nil {
return nil, errors.Wrapf(err, "parseContainerVolumeMount %s", vmStr)
}
vms = append(vms, vm)
}
params := &computeapi.ServerCreateInput{
ServerConfigs: config,
VcpuCount: o.VcpuCount,
Pod: &computeapi.PodCreateInput{
PortMappings: portMappings,
Containers: []*computeapi.PodContainerCreateInput{
{
ContainerSpec: computeapi.ContainerSpec{
ContainerSpec: apis.ContainerSpec{
Image: o.IMAGE,
Command: o.Command,
Args: o.Args,
WorkingDir: o.WorkingDir,
Envs: envs,
EnableLxcfs: o.EnableLxcfs,
VolumeMounts: vms,
},
Devices: devs,
},
},
},
},
}
if options.BoolV(o.AllowDelete) {
disableDelete := false
params.DisableDelete = &disableDelete
}
if regutils.MatchSize(o.MEM) {
memSize, err := fileutils.GetSizeMb(o.MEM, 'M', 1024)
if err != nil {
return nil, err
}
params.VmemSize = memSize
} else {
return nil, fmt.Errorf("Invalid memory input: %q", o.MEM)
}
for idx := range o.IsolatedDevice {
tmpIdx := idx
params.Pod.Containers[0].Devices = append(
params.Pod.Containers[0].Devices,
&computeapi.ContainerDevice{
Type: apis.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE,
IsolatedDevice: &computeapi.ContainerIsolatedDevice{Index: &tmpIdx},
})
}
params.OsArch = o.Arch
params.Name = o.NAME
return params, nil
}

62
pkg/mcclient/options/compute/server_pod_test.go Normal file
View File

@@ -0,0 +1,62 @@
package compute
import (
"reflect"
"testing"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
)
func Test_parsePodPortMapping(t *testing.T) {
tests := []struct {
args string
want *computeapi.PodPortMapping
wantErr bool
}{
{
args: "80:8080/tcp",
want: &computeapi.PodPortMapping{
Protocol: computeapi.PodPortMappingProtocolTCP,
ContainerPort: 8080,
HostPort: 80,
},
wantErr: false,
},
{
args: "80:8080",
want: &computeapi.PodPortMapping{
Protocol: computeapi.PodPortMappingProtocolTCP,
ContainerPort: 8080,
HostPort: 80,
},
wantErr: false,
},
{
args: "80:8080:tcp",
want: nil,
wantErr: true,
},
{
args: "80",
want: nil,
wantErr: true,
},
{
args: "80s:ctrP",
want: nil,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.args, func(t *testing.T) {
got, err := parsePodPortMapping(tt.args)
if (err != nil) != tt.wantErr {
t.Errorf("parsePodPortMapping() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parsePodPortMapping() got = %v, want %v", got, tt.want)
}
})
}
}

Some files were not shown because too many files have changed in this diff Show More