feat(region,host): container mps support (#20138)

* feat(region,host): container mps support

* fix: gencopyright

build/mps-daemon/root/usr/lib/systemd/system/yunion-mps-daemon.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Yunion MPS Daemon
+Documentation=https://docs.yunion.cn
+
+[Service]
+Type=simple
+User=root
+Group=root
+ExecStart=/opt/yunion/bin/mps-daemon
+WorkingDirectory=/opt/yunion/bin
+KillMode=process
+Restart=always
+RestartSec=30
+LimitNOFILE=500000
+LimitNPROC=500000
+
+[Install]
+WantedBy=multi-user.target

build/mps-daemon/vars

@@ -0,0 +1,2 @@
+DESCRIPTION="Yunion MPS Daemon Utility"
+SERVICE="yes"

cmd/mps-daemon/main.go

@@ -0,0 +1,228 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/signalutils"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/util/sysutils"
+)
+
+var mpsControlBin = "nvidia-cuda-mps-control"
+
+type Daemon struct {
+	logDir  string
+	pipeDir string
+
+	replicas int
+}
+
+func NewDaemon(logDir, pipeDir string, replicas int) (*Daemon, error) {
+	if err := os.MkdirAll(pipeDir, 0755); err != nil {
+		return nil, fmt.Errorf("error creating directory %v: %s", pipeDir, err)
+	}
+
+	if err := os.MkdirAll(logDir, 0755); err != nil {
+		return nil, fmt.Errorf("error creating directory %v: %s", logDir, err)
+	}
+
+	return &Daemon{
+		logDir:  logDir,
+		pipeDir: pipeDir,
+
+		replicas: replicas,
+	}, nil
+}
+
+type envvars map[string]string
+
+func (e envvars) toSlice() []string {
+	var envs []string
+	for k, v := range e {
+		envs = append(envs, k+"="+v)
+	}
+	return envs
+}
+
+func (d *Daemon) LogDir() string {
+	return d.logDir
+}
+
+func (d *Daemon) PipeDir() string {
+	return d.pipeDir
+}
+
+func (d *Daemon) Envvars() envvars {
+	return map[string]string{
+		"CUDA_MPS_PIPE_DIRECTORY": d.PipeDir(),
+		"CUDA_MPS_LOG_DIRECTORY":  d.LogDir(),
+	}
+}
+
+// EchoPipeToControl sends the specified command to the MPS control daemon.
+func (d *Daemon) EchoPipeToControl(command string) (string, error) {
+	var out bytes.Buffer
+	reader, writer := io.Pipe()
+	defer writer.Close()
+	defer reader.Close()
+
+	mpsDaemon := exec.Command(mpsControlBin)
+	mpsDaemon.Env = append(mpsDaemon.Env, d.Envvars().toSlice()...)
+
+	mpsDaemon.Stdin = reader
+	mpsDaemon.Stdout = &out
+
+	if err := mpsDaemon.Start(); err != nil {
+		return "", fmt.Errorf("failed to start NVIDIA MPS command: %w", err)
+	}
+
+	if _, err := writer.Write([]byte(command)); err != nil {
+		return "", fmt.Errorf("failed to write message to pipe: %w", err)
+	}
+	_ = writer.Close()
+
+	if err := mpsDaemon.Wait(); err != nil {
+		return "", fmt.Errorf("failed to send command to MPS daemon: %w", err)
+	}
+	return out.String(), nil
+}
+
+func parseMemSize(memTotalStr string) (int, error) {
+	if !strings.HasSuffix(memTotalStr, " MiB") {
+		return -1, fmt.Errorf("unknown mem string suffix")
+	}
+	memStr := strings.TrimSpace(strings.TrimSuffix(memTotalStr, " MiB"))
+	return strconv.Atoi(memStr)
+}
+
+func (d *Daemon) Start() error {
+	// nvidia-smi  --query-gpu=gpu_uuid,memory.total,compute_mode --format=csv
+	// GPU-76aef7ff-372d-2432-b4b4-beca4d8d3400, 23040 MiB, Exclusive_Process
+	out, err := exec.Command("nvidia-smi", "--query-gpu=index,memory.total,compute_mode", "--format=csv").CombinedOutput()
+	if err != nil {
+		return errors.Wrapf(err, "nvidia-smi failed %s", out)
+	}
+
+	var devices = map[string]int{}
+	lines := strings.Split(string(out), "\n")
+	for _, line := range lines {
+		if strings.HasPrefix(line, "index") {
+			continue
+		}
+		segs := strings.Split(line, ",")
+		if len(segs) != 3 {
+			log.Errorf("unknown nvidia-smi out line %s", line)
+			continue
+		}
+		gpuIdx, memTotal, computeMode := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2])
+		if computeMode != "Exclusive_Process" {
+			output, err := exec.Command("nvidia-smi", "-i", gpuIdx, "-c", "EXCLUSIVE_PROCESS").CombinedOutput()
+			if err != nil {
+				return fmt.Errorf("error running nvidia-smi: %s %s", output, err)
+			}
+		}
+
+		memSize, err := parseMemSize(memTotal)
+		if err != nil {
+			return errors.Wrapf(err, "failed parse memSize %s", memTotal)
+		}
+		devices[gpuIdx] = memSize
+	}
+
+	mpsDaemon := exec.Command(mpsControlBin, "-d")
+	mpsDaemon.Env = append(mpsDaemon.Env, d.Envvars().toSlice()...)
+	if err := mpsDaemon.Run(); err != nil {
+		return err
+	}
+	for deviceIdx, memory := range devices {
+		memLimit := memory / d.replicas
+		memLimitCmd := fmt.Sprintf("set_default_device_pinned_mem_limit %s %dM", deviceIdx, memLimit)
+		log.Infof("set device mem limit cmd: %s", memLimitCmd)
+		_, err := d.EchoPipeToControl(memLimitCmd)
+		if err != nil {
+			return fmt.Errorf("error set_default_device_pinned_mem_limit %s", err)
+		}
+	}
+
+	threadPercentageCmd := fmt.Sprintf("set_default_active_thread_percentage %d", 100/d.replicas)
+	_, err = d.EchoPipeToControl(threadPercentageCmd)
+	if err != nil {
+		return fmt.Errorf("error setting active thread percentage: %s", err)
+	}
+
+	return nil
+}
+
+func (d *Daemon) Stop() error {
+	output, err := d.EchoPipeToControl("quit")
+	if err != nil {
+		return fmt.Errorf("error sending quit message: %s %s", output, err)
+	}
+	return nil
+}
+
+func main() {
+	options.Init()
+	isRoot := sysutils.IsRootPermission()
+	if !isRoot {
+		log.Fatalf("host service must running with root permissions")
+		return
+	}
+
+	daemon, err := NewDaemon(
+		options.HostOptions.CudaMPSLogDirectory,
+		options.HostOptions.CudaMPSPipeDirectory,
+		options.HostOptions.CudaMPSReplicas,
+	)
+	if err != nil {
+		log.Fatalf(err.Error())
+		return
+	}
+
+	var sigChan = make(chan struct{})
+	signalutils.RegisterSignal(func() {
+		utils.DumpAllGoroutineStack(log.Logger().Out)
+	}, syscall.SIGUSR1)
+	signalutils.RegisterSignal(func() {
+		sigChan <- struct{}{}
+	}, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)
+	signalutils.StartTrap()
+
+	if err = daemon.Start(); err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	log.Infof("MPS daemon started ......")
+	select {
+	case <-sigChan:
+		if err := daemon.Stop(); err != nil {
+			log.Errorf("failed stop daemon %s", err)
+			os.Exit(1)
+		}
+	}
+}

pkg/apis/compute/container.go

@@ -35,6 +35,7 @@ const (
 	CONTAINER_DEV_NETINT_CA_ASIC   = "NETINT_CA_ASIC"
 	CONTAINER_DEV_NETINT_CA_QUADRA = "NETINT_CA_QUADRA"
 	CONTAINER_DEV_NVIDIA_GPU       = "NVIDIA_GPU"
+	CONTAINER_DEV_NVIDIA_MPS       = "NVIDIA_MPS"
 )
 
 const (

pkg/apis/compute/isolated_device_const.go

@@ -31,7 +31,10 @@ const (
 var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE}
 var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE}
 
-var VALID_CONTAINER_DEVICE_TYPES = []string{CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA, CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU}
+var VALID_CONTAINER_DEVICE_TYPES = []string{
+	CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA,
+	CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS,
+}
 
 var VALID_PASSTHROUGH_TYPES = []string{
 	DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE,

pkg/cloudid/drivers/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package drivers // import "yunion.io/x/onecloud/pkg/cloudid/drivers"

pkg/cloudid/saml/providers/volcengine/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package volcengine // import "yunion.io/x/onecloud/pkg/cloudid/saml/providers/volcengine"

pkg/compute/container_drivers/device/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package device // import "yunion.io/x/onecloud/pkg/compute/container_drivers/device"

pkg/compute/container_drivers/device/host.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package device
 
 import (

pkg/compute/container_drivers/device/isolated_device.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package device
 
 import (

pkg/compute/container_drivers/lifecycle/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package lifecycle // import "yunion.io/x/onecloud/pkg/compute/container_drivers/lifecycle"

pkg/compute/container_drivers/volume_mount/disk.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package volume_mount
 
 import (

pkg/compute/container_drivers/volume_mount/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package volume_mount // import "yunion.io/x/onecloud/pkg/compute/container_drivers/volume_mount"

pkg/compute/container_drivers/volume_mount/host_local.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package volume_mount
 
 import (

pkg/compute/hostdrivers/container.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package hostdrivers
 
 import (

pkg/compute/models/container_drivers.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/compute/models/guests.go

@@ -4195,7 +4195,7 @@ func (self *SGuest) allocSriovNicDevice(
 	}
 	netConfig.SriovDevice.NetworkIndex = &gn.Index
 	netConfig.SriovDevice.WireId = net.WireId
-	err = self.createIsolatedDeviceOnHost(ctx, userCred, host, netConfig.SriovDevice, pendingUsageZone)
+	err = self.createIsolatedDeviceOnHost(ctx, userCred, host, netConfig.SriovDevice, pendingUsageZone, nil)
 	if err != nil {
 		return errors.Wrap(err, "self.createIsolatedDeviceOnHost")
 	}
@@ -4377,7 +4377,7 @@ func (self *SGuest) attachNVMEDevice(
 ) error {
 	gd := self.GetGuestDisk(disk.Id)
 	diskConfig.NVMEDevice.DiskIndex = &gd.Index
-	err := self.createIsolatedDeviceOnHost(ctx, userCred, host, diskConfig.NVMEDevice, pendingUsage)
+	err := self.createIsolatedDeviceOnHost(ctx, userCred, host, diskConfig.NVMEDevice, pendingUsage, nil)
 	if err != nil {
 		return errors.Wrap(err, "self.createIsolatedDeviceOnHost")
 	}
@@ -4518,11 +4518,12 @@ func (self *SGuest) createDiskOnHost(
 }
 
 func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devs []*api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error {
+	usedDeviceMap := map[string]struct{}{}
 	for _, devConfig := range devs {
 		if devConfig.DevType == api.NIC_TYPE || devConfig.DevType == api.NVME_PT_TYPE {
 			continue
 		}
-		err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage)
+		err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage, usedDeviceMap)
 		if err != nil {
 			return err
 		}
@@ -4530,11 +4531,11 @@ func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcc
 	return nil
 }
 
-func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error {
+func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *api.IsolatedDeviceConfig, pendingUsage quotas.IQuota, usedDevMap map[string]struct{}) error {
 	lockman.LockClass(ctx, QuotaManager, self.ProjectId)
 	defer lockman.ReleaseClass(ctx, QuotaManager, self.ProjectId)
 
-	err := IsolatedDeviceManager.attachHostDeviceToGuestByDesc(ctx, self, host, devConfig, userCred)
+	err := IsolatedDeviceManager.attachHostDeviceToGuestByDesc(ctx, self, host, devConfig, userCred, usedDevMap)
 	if err != nil {
 		return err
 	}

pkg/compute/models/host_isolated_device_models.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/compute/models/isolated_devices.go

@@ -132,6 +132,13 @@ type SIsolatedDevice struct {
 	// The maximum number of vGPU instances per physical GPU
 	MaxInstance string `nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
 
+	// MPS perdevice memory limit MB
+	MpsMemoryLimit int `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"`
+	// MPS device memory total MB
+	MpsMemoryTotal int `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"`
+	// MPS device thread percentage
+	MpsThreadPercentage int `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"`
+
 	VendorDeviceId string `width:"16" charset:"ascii" nullable:"true" list:"domain" create:"domain_optional"`
 
 	// reserved memory size for isolated device
@@ -553,11 +560,11 @@ func (manager *SIsolatedDeviceManager) _isValidDeviceInfo(config *api.IsolatedDe
 	return nil
 }
 
-func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error {
+func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]struct{}) error {
 	if len(devConfig.Id) > 0 {
 		return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred)
 	} else if len(devConfig.DevicePath) > 0 {
-		return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred)
+		return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred, usedDevMap)
 	} else {
 		return manager.attachHostDeviceToGuestByModel(ctx, guest, host, devConfig, userCred)
 	}
@@ -575,7 +582,7 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C
 	return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex)
 }
 
-func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error {
+func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]struct{}) error {
 	if len(devConfig.Model) == 0 || len(devConfig.DevicePath) == 0 {
 		return fmt.Errorf("Model or DevicePath is empty: %#v", devConfig)
 	}
@@ -584,7 +591,16 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c
 	if err != nil || len(devs) == 0 {
 		return fmt.Errorf("Can't found model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id)
 	}
-	selectedDev := devs[0]
+	var selectedDev SIsolatedDevice
+	for i := range devs {
+		if _, ok := usedDevMap[devs[i].DevicePath]; !ok {
+			selectedDev = devs[i]
+			usedDevMap[devs[i].DevicePath] = struct{}{}
+		}
+	}
+	if selectedDev.Id == "" {
+		return fmt.Errorf("Can't found unused model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id)
+	}
 	return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex)
 }
 

pkg/compute/models/pod_driver.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/compute/tasks/container_save_volume_mount_image_task.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package tasks
 
 import (

pkg/compute/tasks/disk_migrate_task.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package tasks
 
 import (

pkg/hostman/container/device/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package device // import "yunion.io/x/onecloud/pkg/hostman/container/device"

pkg/hostman/container/device/host.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package device
 
 import (

pkg/hostman/container/device/interface.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package device
 
 import (

pkg/hostman/container/device/isolated_device.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package device
 
 import (

pkg/hostman/container/lifecycle/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package lifecycle // import "yunion.io/x/onecloud/pkg/hostman/container/lifecycle"

pkg/hostman/container/lifecycle/exec.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package lifecycle
 
 import (

pkg/hostman/container/lifecycle/lifecycle.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package lifecycle
 
 import (

pkg/hostman/container/storage/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package storage // import "yunion.io/x/onecloud/pkg/hostman/container/storage"

pkg/hostman/container/volume_mount/disk.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package volume_mount
 
 import (

pkg/hostman/container/volume_mount/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package volume_mount // import "yunion.io/x/onecloud/pkg/hostman/container/volume_mount"

pkg/hostman/container/volume_mount/host_path.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package volume_mount
 
 import (

pkg/hostman/container/volume_mount/interface.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package volume_mount
 
 import (

pkg/hostman/guestman/pod.go

@@ -1007,13 +1007,8 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 			}
 			ctrCfg.Devices = append(ctrCfg.Devices, ctrDevs...)
 		}
-
-		nvMan, err := isolated_device.GetContainerDeviceManager(isolated_device.ContainerDeviceTypeNVIDIAGPU)
-		if err != nil {
-			return "", errors.Wrapf(err, "GetContainerDeviceManager by type %q", isolated_device.ContainerDeviceTypeNVIDIAGPU)
-		}
-		if envs := nvMan.GetContainerEnvs(spec.Devices); len(envs) > 0 {
-			ctrCfg.Envs = append(ctrCfg.Envs, envs...)
+		if err := s.getIsolatedDeviceExtraConfig(spec, ctrCfg); err != nil {
+			return "", err
 		}
 	}
 	if len(spec.Command) != 0 {
@@ -1032,6 +1027,24 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 	return criId, nil
 }
 
+func (s *sPodGuestInstance) getIsolatedDeviceExtraConfig(spec *hostapi.ContainerSpec, ctrCfg *runtimeapi.ContainerConfig) error {
+	devTypes := []isolated_device.ContainerDeviceType{isolated_device.ContainerDeviceTypeNvidiaGpu, isolated_device.ContainerDeviceTypeNvidiaMps}
+	for _, devType := range devTypes {
+		devMan, err := isolated_device.GetContainerDeviceManager(devType)
+		if err != nil {
+			return errors.Wrapf(err, "GetContainerDeviceManager by type %q", devType)
+		}
+		envs, mounts := devMan.GetContainerExtraConfigures(spec.Devices)
+		if len(envs) > 0 {
+			ctrCfg.Envs = append(ctrCfg.Envs, envs...)
+		}
+		if len(mounts) > 0 {
+			ctrCfg.Mounts = append(ctrCfg.Mounts, mounts...)
+		}
+	}
+	return nil
+}
+
 func (s *sPodGuestInstance) getContainerSystemCpusDir(ctrId string) string {
 	return filepath.Join(s.HomeDir(), "cpus", ctrId)
 }

pkg/hostman/guestman/podhandlers/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package podhandlers // import "yunion.io/x/onecloud/pkg/hostman/guestman/podhandlers"

pkg/hostman/guestman/podhandlers/podhandlers.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package podhandlers
 
 import (

pkg/hostman/guestman/runtime.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package guestman
 
 import (

pkg/hostman/hostinfo/hostinfo.go

@@ -2134,9 +2134,16 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		return nil, err
 	}
 	h.IsolatedDeviceMan.ProbePCIDevices(
-		options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, options.HostOptions.DisableCustomDevice,
-		sriovNics, offloadNics, options.HostOptions.PTNVMEConfigs, options.HostOptions.AMDVgpuPFs, options.HostOptions.NVIDIAVgpuPFs,
-		enableDevWhitelist)
+		options.HostOptions.DisableGPU,
+		options.HostOptions.DisableUSB,
+		options.HostOptions.DisableCustomDevice,
+		sriovNics, offloadNics,
+		options.HostOptions.PTNVMEConfigs,
+		options.HostOptions.AMDVgpuPFs,
+		options.HostOptions.NVIDIAVgpuPFs,
+		options.HostOptions.EnableCudaMPS,
+		enableDevWhitelist,
+	)
 
 	objs, err := h.getRemoteIsolatedDevices()
 	if err != nil {

pkg/hostman/isolated_device/container_device.go

@@ -36,7 +36,8 @@ const (
 	ContainerDeviceTypeCphASOPBinder ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER
 	ContainerNetintCAASIC            ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC
 	ContainerNetintCAQuadra          ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA
-	ContainerDeviceTypeNVIDIAGPU     ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU
+	ContainerDeviceTypeNvidiaGpu     ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU
+	ContainerDeviceTypeNvidiaMps     ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS
 )
 
 func GetContainerDeviceManager(t ContainerDeviceType) (IContainerDeviceManager, error) {
@@ -69,5 +70,5 @@ type IContainerDeviceManager interface {
 	NewDevices(dev *ContainerDevice) ([]IDevice, error)
 	NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error)
 	ProbeDevices() ([]IDevice, error)
-	GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue
+	GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount)
 }

pkg/hostman/isolated_device/container_device/base_dev.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package container_device
 
 import (
@@ -38,6 +52,18 @@ func (c BaseDevice) GetDevicePath() string {
 	return c.Path
 }
 
+func (c BaseDevice) GetNvidiaMpsMemoryLimit() int {
+	return -1
+}
+
+func (c BaseDevice) GetNvidiaMpsMemoryTotal() int {
+	return -1
+}
+
+func (c BaseDevice) GetNvidiaMpsThreadPercentage() int {
+	return -1
+}
+
 func CheckVirtualNumber(dev *isolated_device.ContainerDevice) error {
 	if dev.VirtualNumber <= 0 {
 		return errors.Errorf("virtual_number must > 0")

pkg/hostman/isolated_device/container_device/cph_amd_gpu.go

@@ -81,8 +81,8 @@ func (m *cphAMDGPUManager) NewContainerDevices(_ *hostapi.ContainerCreateInput,
 	return []*runtimeapi.Device{cDev}, nil
 }
 
-func (m *cphAMDGPUManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue {
-	return nil
+func (m *cphAMDGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	return nil, nil
 }
 
 type cphAMDGPU struct {

pkg/hostman/isolated_device/container_device/cph_amd_gpu_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package container_device
 
 import "testing"

pkg/hostman/isolated_device/container_device/cph_aosp_binder.go

@@ -123,8 +123,8 @@ func (m *cphAOSPBinderManager) NewContainerDevices(ctrInput *hostapi.ContainerCr
 	return ctrDevs, nil
 }
 
-func (m *cphAOSPBinderManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue {
-	return nil
+func (m *cphAOSPBinderManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	return nil, nil
 }
 
 func (m *cphAOSPBinderManager) ensureBinderDeviceOldWay(dev *hostapi.ContainerIsolatedDevice) error {

pkg/hostman/isolated_device/container_device/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package container_device // import "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device"

pkg/hostman/isolated_device/container_device/net_int_device.go

@@ -166,8 +166,8 @@ func (m *netintDeviceManager) NewContainerDevices(_ *hostapi.ContainerCreateInpu
 	return ctrDevs, nil
 }
 
-func (m *netintDeviceManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue {
-	return nil
+func (m *netintDeviceManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	return nil, nil
 }
 
 type netintDevice struct {

pkg/hostman/isolated_device/container_device/nvidia_gpu.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package container_device
 
 import (
@@ -8,7 +22,6 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
-	"yunion.io/x/onecloud/pkg/apis/host"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	"yunion.io/x/onecloud/pkg/util/procutils"
@@ -25,7 +38,7 @@ func newNvidiaGPUManager() *nvidiaGPUManager {
 }
 
 func (m *nvidiaGPUManager) GetType() isolated_device.ContainerDeviceType {
-	return isolated_device.ContainerDeviceTypeNVIDIAGPU
+	return isolated_device.ContainerDeviceTypeNvidiaGpu
 }
 
 func (m *nvidiaGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
@@ -40,19 +53,19 @@ func (m *nvidiaGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInp
 	return nil, nil
 }
 
-func (m *nvidiaGPUManager) GetContainerEnvs(devs []*host.ContainerDevice) []*runtimeapi.KeyValue {
+func (m *nvidiaGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
 	gpuIds := []string{}
 	for _, dev := range devs {
 		if dev.IsolatedDevice == nil {
 			continue
 		}
-		if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNVIDIAGPU {
+		if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNvidiaGpu {
 			continue
 		}
 		gpuIds = append(gpuIds, dev.IsolatedDevice.Path)
 	}
 	if len(gpuIds) == 0 {
-		return nil
+		return nil, nil
 	}
 
 	return []*runtimeapi.KeyValue{
@@ -64,7 +77,7 @@ func (m *nvidiaGPUManager) GetContainerEnvs(devs []*host.ContainerDevice) []*run
 			Key:   "NVIDIA_DRIVER_CAPABILITIES",
 			Value: "all",
 		},
-	}
+	}, nil
 }
 
 type nvidiaGPU struct {
@@ -76,7 +89,9 @@ func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
 	// nvidia-smi --query-gpu=gpu_uuid,gpu_name,gpu_bus_id --format=csv
 	// uuid, name, pci.bus_id
 	// GPU-bc1a3bb9-55cb-8c52-c374-4f8b4f388a20, NVIDIA A800-SXM4-80GB, 00000000:10:00.0
-	out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id", "--format=csv").Output()
+
+	// nvidia-smi  --query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode --format=csv
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id,compute_mode", "--format=csv").Output()
 	if err != nil {
 		return nil, errors.Wrap(err, "nvidia-smi")
 	}
@@ -90,14 +105,19 @@ func getNvidiaGPUs() ([]isolated_device.IDevice, error) {
 			log.Errorf("unknown nvidia-smi out line %s", line)
 			continue
 		}
-		gpuId, gpuName, gpuPciAddr := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2])
+		gpuId, gpuName, gpuPciAddr, computeMode := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]), strings.TrimSpace(segs[3])
+		if computeMode != "Default" {
+			log.Warningf("gpu device %s compute mode %s, skip.", gpuId, computeMode)
+			continue
+		}
+
 		pciOutput, err := isolated_device.GetPCIStrByAddr(gpuPciAddr)
 		if err != nil {
 			return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", gpuPciAddr)
 		}
 		dev := isolated_device.NewPCIDevice2(pciOutput[0])
 		gpuDev := &nvidiaGPU{
-			BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNVIDIAGPU, gpuId),
+			BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNvidiaGpu, gpuId),
 		}
 		gpuDev.SetModelName(gpuName)
 

pkg/hostman/isolated_device/container_device/nvidia_mps.go

@@ -0,0 +1,187 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+// The MPS /dev/shm is needed to allow MPS daemon health-checking
+var shmPath = "/dev/shm"
+
+func init() {
+	isolated_device.RegisterContainerDeviceManager(newNvidiaMPSManager())
+}
+
+type nvidiaMPSManager struct{}
+
+func newNvidiaMPSManager() *nvidiaMPSManager {
+	return &nvidiaMPSManager{}
+}
+
+func (m *nvidiaMPSManager) GetType() isolated_device.ContainerDeviceType {
+	return isolated_device.ContainerDeviceTypeNvidiaMps
+}
+
+func (m *nvidiaMPSManager) ProbeDevices() ([]isolated_device.IDevice, error) {
+	return getNvidiaMPSGpus()
+}
+
+func (m *nvidiaMPSManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
+	return nil, nil
+}
+
+func (m *nvidiaMPSManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) {
+	return nil, nil
+}
+
+func (m *nvidiaMPSManager) getMPSPipeDirectory() string {
+	return options.HostOptions.CudaMPSPipeDirectory
+}
+
+func (m *nvidiaMPSManager) getSHMPath() string {
+	return shmPath
+}
+
+func (m *nvidiaMPSManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	gpuIds := []string{}
+	for _, dev := range devs {
+		if dev.IsolatedDevice == nil {
+			continue
+		}
+		if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNvidiaMps {
+			continue
+		}
+		gpuIds = append(gpuIds, dev.IsolatedDevice.Path)
+	}
+	if len(gpuIds) == 0 {
+		return nil, nil
+	}
+
+	return []*runtimeapi.KeyValue{
+			{
+				Key:   "CUDA_MPS_PIPE_DIRECTORY",
+				Value: m.getMPSPipeDirectory(),
+			},
+			{
+				Key:   "NVIDIA_VISIBLE_DEVICES",
+				Value: strings.Join(gpuIds, ","),
+			},
+			{
+				Key:   "NVIDIA_DRIVER_CAPABILITIES",
+				Value: "all",
+			},
+		}, []*runtimeapi.Mount{
+			{
+				ContainerPath: m.getSHMPath(),
+				HostPath:      m.getSHMPath(),
+			},
+			{
+				ContainerPath: m.getMPSPipeDirectory(),
+				HostPath:      m.getMPSPipeDirectory(),
+			},
+		}
+}
+
+type nvidiaMPS struct {
+	*BaseDevice
+
+	MemSizeMB        int
+	MemTotalMB       int
+	ThreadPercentage int
+}
+
+func (c *nvidiaMPS) GetNvidiaMpsMemoryLimit() int {
+	return c.MemSizeMB
+}
+
+func (c *nvidiaMPS) GetNvidiaMpsMemoryTotal() int {
+	return c.MemTotalMB
+}
+
+func (c *nvidiaMPS) GetNvidiaMpsThreadPercentage() int {
+	return c.ThreadPercentage
+}
+
+func parseMemSize(memTotalStr string) (int, error) {
+	if !strings.HasSuffix(memTotalStr, " MiB") {
+		return -1, errors.Errorf("unknown mem string suffix")
+	}
+	memStr := strings.TrimSpace(strings.TrimSuffix(memTotalStr, " MiB"))
+	return strconv.Atoi(memStr)
+}
+
+func getNvidiaMPSGpus() ([]isolated_device.IDevice, error) {
+	devs := make([]isolated_device.IDevice, 0)
+	// nvidia-smi  --query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode --format=csv
+	// GPU-76aef7ff-372d-2432-b4b4-beca4d8d3400, Tesla P40, 00000000:00:08.0, 23040 MiB, Exclusive_Process
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode", "--format=csv").Output()
+	if err != nil {
+		return nil, errors.Wrap(err, "nvidia-smi")
+	}
+	lines := strings.Split(string(out), "\n")
+	for _, line := range lines {
+		if strings.HasPrefix(line, "uuid") {
+			continue
+		}
+		segs := strings.Split(line, ",")
+		if len(segs) != 5 {
+			log.Errorf("unknown nvidia-smi out line %s", line)
+			continue
+		}
+		gpuId, gpuName, gpuPciAddr, memTotal, computeMode := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]), strings.TrimSpace(segs[3]), strings.TrimSpace(segs[4])
+		if computeMode != "Exclusive_Process" {
+			log.Warningf("gpu device %s compute mode %s, skip.", gpuId, computeMode)
+			continue
+		}
+		memSize, err := parseMemSize(memTotal)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed parse memSize %s", memTotal)
+		}
+
+		pciOutput, err := isolated_device.GetPCIStrByAddr(gpuPciAddr)
+		if err != nil {
+			return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", gpuPciAddr)
+		}
+		for i := 0; i < options.HostOptions.CudaMPSReplicas; i++ {
+			dev := isolated_device.NewPCIDevice2(pciOutput[0])
+			gpuDev := &nvidiaMPS{
+				BaseDevice:       NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNvidiaMps, gpuId),
+				MemSizeMB:        memSize / options.HostOptions.CudaMPSReplicas,
+				MemTotalMB:       memSize,
+				ThreadPercentage: 100 / options.HostOptions.CudaMPSReplicas,
+			}
+			gpuDev.SetModelName(gpuName)
+			gpuDev.SetAddr(fmt.Sprintf("%s-%d", gpuDev.GetAddr(), i))
+			devs = append(devs, gpuDev)
+		}
+	}
+	if len(devs) == 0 {
+		return nil, nil
+	}
+	return devs, nil
+}

pkg/hostman/isolated_device/isolated_device.go

@@ -111,13 +111,18 @@ type IDevice interface {
 	// Get extra PCIE information
 	GetPCIEInfo() *api.IsolatedDevicePCIEInfo
 	GetDevicePath() string
+
+	// mps infos
+	GetNvidiaMpsMemoryLimit() int
+	GetNvidiaMpsMemoryTotal() int
+	GetNvidiaMpsThreadPercentage() int
 }
 
 type IsolatedDeviceManager interface {
 	GetDevices() []IDevice
 	GetDeviceByIdent(vendorDevId, addr, mdevId string) IDevice
 	GetDeviceByAddr(addr string) IDevice
-	ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool)
+	ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableCudaMps, enableWhitelist bool)
 	StartDetachTask()
 	BatchCustomProbe()
 	AppendDetachedDevice(dev *CloudDeviceInfo)
@@ -188,10 +193,15 @@ func (man *isolatedDeviceManager) probeContainerDevices() {
 	}
 }
 
-func (man *isolatedDeviceManager) probeContainerNvidiaGPUs() {
-	devman, err := GetContainerDeviceManager(ContainerDeviceTypeNVIDIAGPU)
+func (man *isolatedDeviceManager) probeContainerNvidiaGPUs(enableCudaMps bool) {
+	devType := ContainerDeviceTypeNvidiaGpu
+	if enableCudaMps {
+		devType = ContainerDeviceTypeNvidiaMps
+	}
+
+	devman, err := GetContainerDeviceManager(devType)
 	if err != nil {
-		log.Errorf("no container device manager %s found", ContainerDeviceTypeNVIDIAGPU)
+		log.Errorf("no container device manager %s found", devType)
 		return
 	}
 	devs, err := devman.ProbeDevices()
@@ -369,10 +379,15 @@ func (man *isolatedDeviceManager) probeNVIDIAVgpus(nvidiaVgpuPFs []string) {
 	}
 }
 
-func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool) {
+func (man *isolatedDeviceManager) ProbePCIDevices(
+	skipGPUs, skipUSBs, skipCustomDevs bool,
+	sriovNics, ovsOffloadNics []HostNic,
+	nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string,
+	enableCudaMps, enableWhitelist bool,
+) {
 	man.devices = make([]IDevice, 0)
 	if man.host.IsContainerHost() {
-		man.probeContainerNvidiaGPUs()
+		man.probeContainerNvidiaGPUs(enableCudaMps)
 		man.probeContainerDevices()
 	} else {
 		devModels, err := man.getCustomIsolatedDeviceModels()
@@ -569,6 +584,10 @@ func (dev *SBaseDevice) GetAddr() string {
 	return dev.dev.Addr
 }
 
+func (dev *SBaseDevice) SetAddr(addr string) {
+	dev.dev.Addr = addr
+}
+
 func (dev *SBaseDevice) GetDeviceType() string {
 	return dev.devType
 }
@@ -628,6 +647,18 @@ func (dev *SBaseDevice) GetGuestId() string {
 	return dev.guestId
 }
 
+func (dev *SBaseDevice) GetNvidiaMpsMemoryLimit() int {
+	return -1
+}
+
+func (dev *SBaseDevice) GetNvidiaMpsMemoryTotal() int {
+	return -1
+}
+
+func (dev *SBaseDevice) GetNvidiaMpsThreadPercentage() int {
+	return -1
+}
+
 func GetApiResourceData(dev IDevice) *jsonutils.JSONDict {
 	data := map[string]interface{}{
 		"dev_type":         dev.GetDeviceType(),
@@ -682,6 +713,16 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict {
 	if devPath != "" {
 		data["device_path"] = devPath
 	}
+
+	if mpsMemTotal := dev.GetNvidiaMpsMemoryTotal(); mpsMemTotal > 0 {
+		data["mps_memory_total"] = mpsMemTotal
+	}
+	if mpsMemLimit := dev.GetNvidiaMpsMemoryLimit(); mpsMemLimit > 0 {
+		data["mps_memory_limit"] = mpsMemLimit
+	}
+	if mpsThreadPercentage := dev.GetNvidiaMpsThreadPercentage(); mpsThreadPercentage > 0 {
+		data["mps_thread_percentage"] = mpsThreadPercentage
+	}
 	return jsonutils.Marshal(data).(*jsonutils.JSONDict)
 }
 

pkg/hostman/isolated_device/nvidia_vgpu.go

@@ -101,6 +101,18 @@ func (dev *sNVIDIAVgpuDevice) GetDevicePath() string {
 	return ""
 }
 
+func (dev *sNVIDIAVgpuDevice) GetNvidiaMpsMemoryLimit() int {
+	return -1
+}
+
+func (dev *sNVIDIAVgpuDevice) GetNvidiaMpsMemoryTotal() int {
+	return -1
+}
+
+func (dev *sNVIDIAVgpuDevice) GetNvidiaMpsThreadPercentage() int {
+	return -1
+}
+
 func (dev *sNVIDIAVgpuDevice) SetDeviceInfo(info CloudDeviceInfo) {
 	if len(info.Id) != 0 {
 		dev.cloudId = info.Id

pkg/hostman/options/options.go

@@ -212,6 +212,11 @@ type SHostOptions struct {
 	ContainerRuntimeEndpoint  string `help:"endpoint of container runtime service" default:"unix:///var/run/onecloud/containerd/containerd.sock"`
 	ContainerDeviceConfigFile string `help:"container device configuration file path"`
 	LxcfsPath                 string `help:"lxcfs directory path" default:"/var/lib/lxcfs"`
+
+	EnableCudaMPS        bool   `help:"enable cuda mps" default:"false"`
+	CudaMPSPipeDirectory string `help:"cuda mps pipe dir" default:"/tmp/nvidia-mps/pipe"`
+	CudaMPSLogDirectory  string `help:"cuda mps log dir" default:"/tmp/nvidia-mps/log"`
+	CudaMPSReplicas      int    `help:"cuda mps replias" default:"10"`
 }
 
 var (

pkg/hostman/storageman/container_storage/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package container_storage // import "yunion.io/x/onecloud/pkg/hostman/storageman/container_storage"

pkg/hostman/storageman/container_storage/local_raw.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package container_storage
 
 import (
@@ -57,8 +71,8 @@ func (m *localLoopDiskManager) ProbeDevices() ([]isolated_device.IDevice, error)
 	return nil, nil
 }
 
-func (m *localLoopDiskManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue {
-	return nil
+func (m *localLoopDiskManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	return nil, nil
 }
 
 func newLocalLoopDiskManager() *localLoopDiskManager {

pkg/mcclient/modules/compute/mod_host_isolated_device_models.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 import (

pkg/mcclient/options/compute/server_pod_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 import (

pkg/scheduler/algorithm/predicates/isolated_device_predicate.go

@@ -56,6 +56,19 @@ func (f *IsolatedDevicePredicate) PreExecute(ctx context.Context, u *core.Unit,
 	return false, nil
 }
 
+func (f *IsolatedDevicePredicate) getIsolatedDeviceCountByType(getter core.CandidatePropertyGetter, devType string) int {
+	devs := getter.UnusedIsolatedDevicesByType(devType)
+	if devType != compute.CONTAINER_DEV_NVIDIA_MPS {
+		return len(devs)
+	} else {
+		devMap := map[string]struct{}{}
+		for _, dev := range devs {
+			devMap[dev.DevicePath] = struct{}{}
+		}
+		return len(devMap)
+	}
+}
+
 func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
 	h := NewPredicateHelper(f, u, c)
 	reqIsoDevs := u.SchedData().IsolatedDevices
@@ -116,7 +129,7 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c
 		}
 	}
 	for devType, reqCount := range devTypeRequest {
-		freeCount := len(getter.UnusedIsolatedDevicesByType(devType))
+		freeCount := f.getIsolatedDeviceCountByType(getter, devType)
 		if freeCount < reqCount {
 			h.Exclude(fmt.Sprintf("IsolatedDevice type %q not enough, request: %d, hostFree: %d", devType, reqCount, freeCount))
 			return h.GetResult()

pkg/scheduler/data_manager/common/cache_manager.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package common
 
 var cacheManager CacheManager

pkg/util/httpstream/doc.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/httpstream/httpstream.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/httpstream/httpstream_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/httpstream/spdy/connection.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/httpstream/spdy/connection_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/httpstream/spdy/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package spdy // import "yunion.io/x/onecloud/pkg/util/httpstream/spdy"

pkg/util/httpstream/spdy/roundtripper.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/httpstream/spdy/roundtripper_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/httpstream/spdy/upgrade.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/httpstream/spdy/upgrade_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/interrupt/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package interrupt // import "yunion.io/x/onecloud/pkg/util/interrupt"

pkg/util/interrupt/interrupt.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/losetup/device.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package losetup
 
 import (

pkg/util/losetup/device_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package losetup
 
 import (

pkg/util/losetup/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package losetup // import "yunion.io/x/onecloud/pkg/util/losetup"

pkg/util/losetup/losetup.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package losetup
 
 import (

pkg/util/netutils2/getport/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package getport // import "yunion.io/x/onecloud/pkg/util/netutils2/getport"

pkg/util/netutils2/getport/getport_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package getport
 
 import (

pkg/util/pod/cpus.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package pod
 
 import (

pkg/util/pod/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package pod // import "yunion.io/x/onecloud/pkg/util/pod"

pkg/util/pod/pod.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package pod
 
 import (

pkg/util/pod/remotecommand/constants.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/pod/remotecommand/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package remotecommand // import "yunion.io/x/onecloud/pkg/util/pod/remotecommand"

pkg/util/pod/remotecommand/errorstream.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/pod/remotecommand/executor.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package remotecommand
 
 import (

pkg/util/pod/remotecommand/reader.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package remotecommand
 
 import (

pkg/util/pod/remotecommand/remotecommand.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package remotecommand
 
 import (

pkg/util/pod/remotecommand/resize.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package remotecommand
 
 // TerminalSize and TerminalSizeQueue was a port of k8s.io/kubernetes/pkg/util/term

pkg/util/pod/remotecommand/spdy/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package spdy // import "yunion.io/x/onecloud/pkg/util/pod/remotecommand/spdy"

pkg/util/pod/remotecommand/spdy/spdy.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package spdy
 
 import (

pkg/util/pod/remotecommand/transport.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package remotecommand

pkg/util/pod/remotecommand/v1.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/pod/remotecommand/v2.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2015 The Kubernetes Authors.
 

pkg/util/pod/remotecommand/v2_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/pod/remotecommand/v3.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/pod/remotecommand/v4.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/pod/term/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package term // import "yunion.io/x/onecloud/pkg/util/pod/term"

pkg/util/pod/term/resize.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/pod/term/resizeevents.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //go:build !windows
 // +build !windows
 

pkg/util/pod/term/resizeevents_windows.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/pod/term/term.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package term
 
 import (

pkg/util/pod/term/term_writer.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.
 

pkg/util/pod/term/term_writer_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 /*
 Copyright 2016 The Kubernetes Authors.