From 5fc42614c3da3cd8f49b05de0a0e26f2121f378c Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Tue, 30 Apr 2024 17:39:40 +0800 Subject: [PATCH] feat(region,host): container mps support (#20138) * feat(region,host): container mps support * fix: gencopyright --- .../systemd/system/yunion-mps-daemon.service | 18 ++ build/mps-daemon/vars | 2 + cmd/mps-daemon/main.go | 228 ++++++++++++++++++ pkg/apis/compute/container.go | 1 + pkg/apis/compute/isolated_device_const.go | 5 +- pkg/cloudid/drivers/doc.go | 14 ++ pkg/cloudid/saml/providers/volcengine/doc.go | 14 ++ pkg/compute/container_drivers/device/doc.go | 14 ++ pkg/compute/container_drivers/device/host.go | 14 ++ .../device/isolated_device.go | 14 ++ .../container_drivers/lifecycle/doc.go | 14 ++ .../container_drivers/volume_mount/disk.go | 14 ++ .../container_drivers/volume_mount/doc.go | 14 ++ .../volume_mount/host_local.go | 14 ++ pkg/compute/hostdrivers/container.go | 14 ++ pkg/compute/models/container_drivers.go | 14 ++ pkg/compute/models/guests.go | 11 +- .../models/host_isolated_device_models.go | 14 ++ pkg/compute/models/isolated_devices.go | 24 +- pkg/compute/models/pod_driver.go | 14 ++ .../container_save_volume_mount_image_task.go | 14 ++ pkg/compute/tasks/disk_migrate_task.go | 14 ++ pkg/hostman/container/device/doc.go | 14 ++ pkg/hostman/container/device/host.go | 14 ++ pkg/hostman/container/device/interface.go | 14 ++ .../container/device/isolated_device.go | 14 ++ pkg/hostman/container/lifecycle/doc.go | 14 ++ pkg/hostman/container/lifecycle/exec.go | 14 ++ pkg/hostman/container/lifecycle/lifecycle.go | 14 ++ pkg/hostman/container/storage/doc.go | 14 ++ pkg/hostman/container/volume_mount/disk.go | 14 ++ pkg/hostman/container/volume_mount/doc.go | 14 ++ .../container/volume_mount/host_path.go | 14 ++ .../container/volume_mount/interface.go | 14 ++ pkg/hostman/guestman/pod.go | 27 ++- pkg/hostman/guestman/podhandlers/doc.go | 14 ++ .../guestman/podhandlers/podhandlers.go | 14 ++ pkg/hostman/guestman/runtime.go | 14 ++ pkg/hostman/hostinfo/hostinfo.go | 13 +- .../isolated_device/container_device.go | 5 +- .../container_device/base_dev.go | 26 ++ .../container_device/cph_amd_gpu.go | 4 +- .../container_device/cph_amd_gpu_test.go | 14 ++ .../container_device/cph_aosp_binder.go | 4 +- .../isolated_device/container_device/doc.go | 14 ++ .../container_device/net_int_device.go | 4 +- .../container_device/nvidia_gpu.go | 38 ++- .../container_device/nvidia_mps.go | 187 ++++++++++++++ .../isolated_device/isolated_device.go | 53 +++- pkg/hostman/isolated_device/nvidia_vgpu.go | 12 + pkg/hostman/options/options.go | 5 + .../storageman/container_storage/doc.go | 14 ++ .../storageman/container_storage/local_raw.go | 18 +- .../mod_host_isolated_device_models.go | 14 ++ .../options/compute/server_pod_test.go | 14 ++ .../predicates/isolated_device_predicate.go | 15 +- .../data_manager/common/cache_manager.go | 14 ++ pkg/util/httpstream/doc.go | 14 ++ pkg/util/httpstream/httpstream.go | 14 ++ pkg/util/httpstream/httpstream_test.go | 14 ++ pkg/util/httpstream/spdy/connection.go | 14 ++ pkg/util/httpstream/spdy/connection_test.go | 14 ++ pkg/util/httpstream/spdy/doc.go | 14 ++ pkg/util/httpstream/spdy/roundtripper.go | 14 ++ pkg/util/httpstream/spdy/roundtripper_test.go | 14 ++ pkg/util/httpstream/spdy/upgrade.go | 14 ++ pkg/util/httpstream/spdy/upgrade_test.go | 14 ++ pkg/util/interrupt/doc.go | 14 ++ pkg/util/interrupt/interrupt.go | 14 ++ pkg/util/losetup/device.go | 14 ++ pkg/util/losetup/device_test.go | 14 ++ pkg/util/losetup/doc.go | 14 ++ pkg/util/losetup/losetup.go | 14 ++ pkg/util/netutils2/getport/doc.go | 14 ++ pkg/util/netutils2/getport/getport_test.go | 14 ++ pkg/util/pod/cpus.go | 14 ++ pkg/util/pod/doc.go | 14 ++ pkg/util/pod/pod.go | 14 ++ pkg/util/pod/remotecommand/constants.go | 14 ++ pkg/util/pod/remotecommand/doc.go | 14 ++ pkg/util/pod/remotecommand/errorstream.go | 14 ++ pkg/util/pod/remotecommand/executor.go | 14 ++ pkg/util/pod/remotecommand/reader.go | 14 ++ pkg/util/pod/remotecommand/remotecommand.go | 14 ++ pkg/util/pod/remotecommand/resize.go | 14 ++ pkg/util/pod/remotecommand/spdy/doc.go | 14 ++ pkg/util/pod/remotecommand/spdy/spdy.go | 14 ++ pkg/util/pod/remotecommand/transport.go | 14 ++ pkg/util/pod/remotecommand/v1.go | 14 ++ pkg/util/pod/remotecommand/v2.go | 14 ++ pkg/util/pod/remotecommand/v2_test.go | 14 ++ pkg/util/pod/remotecommand/v3.go | 14 ++ pkg/util/pod/remotecommand/v4.go | 14 ++ pkg/util/pod/term/doc.go | 14 ++ pkg/util/pod/term/resize.go | 14 ++ pkg/util/pod/term/resizeevents.go | 14 ++ pkg/util/pod/term/resizeevents_windows.go | 14 ++ pkg/util/pod/term/term.go | 14 ++ pkg/util/pod/term/term_writer.go | 14 ++ pkg/util/pod/term/term_writer_test.go | 14 ++ 100 files changed, 1760 insertions(+), 46 deletions(-) create mode 100644 build/mps-daemon/root/usr/lib/systemd/system/yunion-mps-daemon.service create mode 100644 build/mps-daemon/vars create mode 100644 cmd/mps-daemon/main.go create mode 100644 pkg/hostman/isolated_device/container_device/nvidia_mps.go diff --git a/build/mps-daemon/root/usr/lib/systemd/system/yunion-mps-daemon.service b/build/mps-daemon/root/usr/lib/systemd/system/yunion-mps-daemon.service new file mode 100644 index 0000000000..fda5da815a --- /dev/null +++ b/build/mps-daemon/root/usr/lib/systemd/system/yunion-mps-daemon.service @@ -0,0 +1,18 @@ +[Unit] +Description=Yunion MPS Daemon +Documentation=https://docs.yunion.cn + +[Service] +Type=simple +User=root +Group=root +ExecStart=/opt/yunion/bin/mps-daemon +WorkingDirectory=/opt/yunion/bin +KillMode=process +Restart=always +RestartSec=30 +LimitNOFILE=500000 +LimitNPROC=500000 + +[Install] +WantedBy=multi-user.target diff --git a/build/mps-daemon/vars b/build/mps-daemon/vars new file mode 100644 index 0000000000..512651df47 --- /dev/null +++ b/build/mps-daemon/vars @@ -0,0 +1,2 @@ +DESCRIPTION="Yunion MPS Daemon Utility" +SERVICE="yes" \ No newline at end of file diff --git a/cmd/mps-daemon/main.go b/cmd/mps-daemon/main.go new file mode 100644 index 0000000000..aee45f6294 --- /dev/null +++ b/cmd/mps-daemon/main.go @@ -0,0 +1,228 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/signalutils" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/sysutils" +) + +var mpsControlBin = "nvidia-cuda-mps-control" + +type Daemon struct { + logDir string + pipeDir string + + replicas int +} + +func NewDaemon(logDir, pipeDir string, replicas int) (*Daemon, error) { + if err := os.MkdirAll(pipeDir, 0755); err != nil { + return nil, fmt.Errorf("error creating directory %v: %s", pipeDir, err) + } + + if err := os.MkdirAll(logDir, 0755); err != nil { + return nil, fmt.Errorf("error creating directory %v: %s", logDir, err) + } + + return &Daemon{ + logDir: logDir, + pipeDir: pipeDir, + + replicas: replicas, + }, nil +} + +type envvars map[string]string + +func (e envvars) toSlice() []string { + var envs []string + for k, v := range e { + envs = append(envs, k+"="+v) + } + return envs +} + +func (d *Daemon) LogDir() string { + return d.logDir +} + +func (d *Daemon) PipeDir() string { + return d.pipeDir +} + +func (d *Daemon) Envvars() envvars { + return map[string]string{ + "CUDA_MPS_PIPE_DIRECTORY": d.PipeDir(), + "CUDA_MPS_LOG_DIRECTORY": d.LogDir(), + } +} + +// EchoPipeToControl sends the specified command to the MPS control daemon. +func (d *Daemon) EchoPipeToControl(command string) (string, error) { + var out bytes.Buffer + reader, writer := io.Pipe() + defer writer.Close() + defer reader.Close() + + mpsDaemon := exec.Command(mpsControlBin) + mpsDaemon.Env = append(mpsDaemon.Env, d.Envvars().toSlice()...) + + mpsDaemon.Stdin = reader + mpsDaemon.Stdout = &out + + if err := mpsDaemon.Start(); err != nil { + return "", fmt.Errorf("failed to start NVIDIA MPS command: %w", err) + } + + if _, err := writer.Write([]byte(command)); err != nil { + return "", fmt.Errorf("failed to write message to pipe: %w", err) + } + _ = writer.Close() + + if err := mpsDaemon.Wait(); err != nil { + return "", fmt.Errorf("failed to send command to MPS daemon: %w", err) + } + return out.String(), nil +} + +func parseMemSize(memTotalStr string) (int, error) { + if !strings.HasSuffix(memTotalStr, " MiB") { + return -1, fmt.Errorf("unknown mem string suffix") + } + memStr := strings.TrimSpace(strings.TrimSuffix(memTotalStr, " MiB")) + return strconv.Atoi(memStr) +} + +func (d *Daemon) Start() error { + // nvidia-smi --query-gpu=gpu_uuid,memory.total,compute_mode --format=csv + // GPU-76aef7ff-372d-2432-b4b4-beca4d8d3400, 23040 MiB, Exclusive_Process + out, err := exec.Command("nvidia-smi", "--query-gpu=index,memory.total,compute_mode", "--format=csv").CombinedOutput() + if err != nil { + return errors.Wrapf(err, "nvidia-smi failed %s", out) + } + + var devices = map[string]int{} + lines := strings.Split(string(out), "\n") + for _, line := range lines { + if strings.HasPrefix(line, "index") { + continue + } + segs := strings.Split(line, ",") + if len(segs) != 3 { + log.Errorf("unknown nvidia-smi out line %s", line) + continue + } + gpuIdx, memTotal, computeMode := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]) + if computeMode != "Exclusive_Process" { + output, err := exec.Command("nvidia-smi", "-i", gpuIdx, "-c", "EXCLUSIVE_PROCESS").CombinedOutput() + if err != nil { + return fmt.Errorf("error running nvidia-smi: %s %s", output, err) + } + } + + memSize, err := parseMemSize(memTotal) + if err != nil { + return errors.Wrapf(err, "failed parse memSize %s", memTotal) + } + devices[gpuIdx] = memSize + } + + mpsDaemon := exec.Command(mpsControlBin, "-d") + mpsDaemon.Env = append(mpsDaemon.Env, d.Envvars().toSlice()...) + if err := mpsDaemon.Run(); err != nil { + return err + } + for deviceIdx, memory := range devices { + memLimit := memory / d.replicas + memLimitCmd := fmt.Sprintf("set_default_device_pinned_mem_limit %s %dM", deviceIdx, memLimit) + log.Infof("set device mem limit cmd: %s", memLimitCmd) + _, err := d.EchoPipeToControl(memLimitCmd) + if err != nil { + return fmt.Errorf("error set_default_device_pinned_mem_limit %s", err) + } + } + + threadPercentageCmd := fmt.Sprintf("set_default_active_thread_percentage %d", 100/d.replicas) + _, err = d.EchoPipeToControl(threadPercentageCmd) + if err != nil { + return fmt.Errorf("error setting active thread percentage: %s", err) + } + + return nil +} + +func (d *Daemon) Stop() error { + output, err := d.EchoPipeToControl("quit") + if err != nil { + return fmt.Errorf("error sending quit message: %s %s", output, err) + } + return nil +} + +func main() { + options.Init() + isRoot := sysutils.IsRootPermission() + if !isRoot { + log.Fatalf("host service must running with root permissions") + return + } + + daemon, err := NewDaemon( + options.HostOptions.CudaMPSLogDirectory, + options.HostOptions.CudaMPSPipeDirectory, + options.HostOptions.CudaMPSReplicas, + ) + if err != nil { + log.Fatalf(err.Error()) + return + } + + var sigChan = make(chan struct{}) + signalutils.RegisterSignal(func() { + utils.DumpAllGoroutineStack(log.Logger().Out) + }, syscall.SIGUSR1) + signalutils.RegisterSignal(func() { + sigChan <- struct{}{} + }, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) + signalutils.StartTrap() + + if err = daemon.Start(); err != nil { + log.Fatalf(err.Error()) + } + + log.Infof("MPS daemon started ......") + select { + case <-sigChan: + if err := daemon.Stop(); err != nil { + log.Errorf("failed stop daemon %s", err) + os.Exit(1) + } + } +} diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index b9ad21ab34..9216a93da6 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -35,6 +35,7 @@ const ( CONTAINER_DEV_NETINT_CA_ASIC = "NETINT_CA_ASIC" CONTAINER_DEV_NETINT_CA_QUADRA = "NETINT_CA_QUADRA" CONTAINER_DEV_NVIDIA_GPU = "NVIDIA_GPU" + CONTAINER_DEV_NVIDIA_MPS = "NVIDIA_MPS" ) const ( diff --git a/pkg/apis/compute/isolated_device_const.go b/pkg/apis/compute/isolated_device_const.go index 25b2e368ae..b4bd893333 100644 --- a/pkg/apis/compute/isolated_device_const.go +++ b/pkg/apis/compute/isolated_device_const.go @@ -31,7 +31,10 @@ const ( var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE} -var VALID_CONTAINER_DEVICE_TYPES = []string{CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA, CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU} +var VALID_CONTAINER_DEVICE_TYPES = []string{ + CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA, + CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, +} var VALID_PASSTHROUGH_TYPES = []string{ DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE, diff --git a/pkg/cloudid/drivers/doc.go b/pkg/cloudid/drivers/doc.go index 512a68c727..1c2afce173 100644 --- a/pkg/cloudid/drivers/doc.go +++ b/pkg/cloudid/drivers/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package drivers // import "yunion.io/x/onecloud/pkg/cloudid/drivers" diff --git a/pkg/cloudid/saml/providers/volcengine/doc.go b/pkg/cloudid/saml/providers/volcengine/doc.go index f70373a8c7..0695aa9ba9 100644 --- a/pkg/cloudid/saml/providers/volcengine/doc.go +++ b/pkg/cloudid/saml/providers/volcengine/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package volcengine // import "yunion.io/x/onecloud/pkg/cloudid/saml/providers/volcengine" diff --git a/pkg/compute/container_drivers/device/doc.go b/pkg/compute/container_drivers/device/doc.go index 93d7e978d3..59264f89de 100644 --- a/pkg/compute/container_drivers/device/doc.go +++ b/pkg/compute/container_drivers/device/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package device // import "yunion.io/x/onecloud/pkg/compute/container_drivers/device" diff --git a/pkg/compute/container_drivers/device/host.go b/pkg/compute/container_drivers/device/host.go index bbc2213aed..0f7a80edd2 100644 --- a/pkg/compute/container_drivers/device/host.go +++ b/pkg/compute/container_drivers/device/host.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package device import ( diff --git a/pkg/compute/container_drivers/device/isolated_device.go b/pkg/compute/container_drivers/device/isolated_device.go index c64af68bf6..d88243a580 100644 --- a/pkg/compute/container_drivers/device/isolated_device.go +++ b/pkg/compute/container_drivers/device/isolated_device.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package device import ( diff --git a/pkg/compute/container_drivers/lifecycle/doc.go b/pkg/compute/container_drivers/lifecycle/doc.go index 7703ece1e8..9d954c0376 100644 --- a/pkg/compute/container_drivers/lifecycle/doc.go +++ b/pkg/compute/container_drivers/lifecycle/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package lifecycle // import "yunion.io/x/onecloud/pkg/compute/container_drivers/lifecycle" diff --git a/pkg/compute/container_drivers/volume_mount/disk.go b/pkg/compute/container_drivers/volume_mount/disk.go index 762782c725..eb2dfa245e 100644 --- a/pkg/compute/container_drivers/volume_mount/disk.go +++ b/pkg/compute/container_drivers/volume_mount/disk.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package volume_mount import ( diff --git a/pkg/compute/container_drivers/volume_mount/doc.go b/pkg/compute/container_drivers/volume_mount/doc.go index 9bdd18110f..1d896a9f1f 100644 --- a/pkg/compute/container_drivers/volume_mount/doc.go +++ b/pkg/compute/container_drivers/volume_mount/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package volume_mount // import "yunion.io/x/onecloud/pkg/compute/container_drivers/volume_mount" diff --git a/pkg/compute/container_drivers/volume_mount/host_local.go b/pkg/compute/container_drivers/volume_mount/host_local.go index 062524e649..2f1514b4e5 100644 --- a/pkg/compute/container_drivers/volume_mount/host_local.go +++ b/pkg/compute/container_drivers/volume_mount/host_local.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package volume_mount import ( diff --git a/pkg/compute/hostdrivers/container.go b/pkg/compute/hostdrivers/container.go index 64c3842bb2..38d2cc4325 100644 --- a/pkg/compute/hostdrivers/container.go +++ b/pkg/compute/hostdrivers/container.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package hostdrivers import ( diff --git a/pkg/compute/models/container_drivers.go b/pkg/compute/models/container_drivers.go index ae4427e252..332a76b81c 100644 --- a/pkg/compute/models/container_drivers.go +++ b/pkg/compute/models/container_drivers.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package models import ( diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 71b7c8c8da..5cc6af1fae 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -4195,7 +4195,7 @@ func (self *SGuest) allocSriovNicDevice( } netConfig.SriovDevice.NetworkIndex = &gn.Index netConfig.SriovDevice.WireId = net.WireId - err = self.createIsolatedDeviceOnHost(ctx, userCred, host, netConfig.SriovDevice, pendingUsageZone) + err = self.createIsolatedDeviceOnHost(ctx, userCred, host, netConfig.SriovDevice, pendingUsageZone, nil) if err != nil { return errors.Wrap(err, "self.createIsolatedDeviceOnHost") } @@ -4377,7 +4377,7 @@ func (self *SGuest) attachNVMEDevice( ) error { gd := self.GetGuestDisk(disk.Id) diskConfig.NVMEDevice.DiskIndex = &gd.Index - err := self.createIsolatedDeviceOnHost(ctx, userCred, host, diskConfig.NVMEDevice, pendingUsage) + err := self.createIsolatedDeviceOnHost(ctx, userCred, host, diskConfig.NVMEDevice, pendingUsage, nil) if err != nil { return errors.Wrap(err, "self.createIsolatedDeviceOnHost") } @@ -4518,11 +4518,12 @@ func (self *SGuest) createDiskOnHost( } func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devs []*api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error { + usedDeviceMap := map[string]struct{}{} for _, devConfig := range devs { if devConfig.DevType == api.NIC_TYPE || devConfig.DevType == api.NVME_PT_TYPE { continue } - err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage) + err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage, usedDeviceMap) if err != nil { return err } @@ -4530,11 +4531,11 @@ func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcc return nil } -func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error { +func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *api.IsolatedDeviceConfig, pendingUsage quotas.IQuota, usedDevMap map[string]struct{}) error { lockman.LockClass(ctx, QuotaManager, self.ProjectId) defer lockman.ReleaseClass(ctx, QuotaManager, self.ProjectId) - err := IsolatedDeviceManager.attachHostDeviceToGuestByDesc(ctx, self, host, devConfig, userCred) + err := IsolatedDeviceManager.attachHostDeviceToGuestByDesc(ctx, self, host, devConfig, userCred, usedDevMap) if err != nil { return err } diff --git a/pkg/compute/models/host_isolated_device_models.go b/pkg/compute/models/host_isolated_device_models.go index 29dc7a266d..6133c2741f 100644 --- a/pkg/compute/models/host_isolated_device_models.go +++ b/pkg/compute/models/host_isolated_device_models.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package models import ( diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index ded6be8c92..6d752740b5 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -132,6 +132,13 @@ type SIsolatedDevice struct { // The maximum number of vGPU instances per physical GPU MaxInstance string `nullable:"true" list:"domain" update:"domain" create:"domain_optional"` + // MPS perdevice memory limit MB + MpsMemoryLimit int `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"` + // MPS device memory total MB + MpsMemoryTotal int `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"` + // MPS device thread percentage + MpsThreadPercentage int `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"` + VendorDeviceId string `width:"16" charset:"ascii" nullable:"true" list:"domain" create:"domain_optional"` // reserved memory size for isolated device @@ -553,11 +560,11 @@ func (manager *SIsolatedDeviceManager) _isValidDeviceInfo(config *api.IsolatedDe return nil } -func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { +func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]struct{}) error { if len(devConfig.Id) > 0 { return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred) } else if len(devConfig.DevicePath) > 0 { - return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred) + return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred, usedDevMap) } else { return manager.attachHostDeviceToGuestByModel(ctx, guest, host, devConfig, userCred) } @@ -575,7 +582,7 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex) } -func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { +func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]struct{}) error { if len(devConfig.Model) == 0 || len(devConfig.DevicePath) == 0 { return fmt.Errorf("Model or DevicePath is empty: %#v", devConfig) } @@ -584,7 +591,16 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c if err != nil || len(devs) == 0 { return fmt.Errorf("Can't found model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id) } - selectedDev := devs[0] + var selectedDev SIsolatedDevice + for i := range devs { + if _, ok := usedDevMap[devs[i].DevicePath]; !ok { + selectedDev = devs[i] + usedDevMap[devs[i].DevicePath] = struct{}{} + } + } + if selectedDev.Id == "" { + return fmt.Errorf("Can't found unused model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id) + } return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex) } diff --git a/pkg/compute/models/pod_driver.go b/pkg/compute/models/pod_driver.go index 11270d1353..30ab65ad17 100644 --- a/pkg/compute/models/pod_driver.go +++ b/pkg/compute/models/pod_driver.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package models import ( diff --git a/pkg/compute/tasks/container_save_volume_mount_image_task.go b/pkg/compute/tasks/container_save_volume_mount_image_task.go index a64b9c1195..95365ae890 100644 --- a/pkg/compute/tasks/container_save_volume_mount_image_task.go +++ b/pkg/compute/tasks/container_save_volume_mount_image_task.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tasks import ( diff --git a/pkg/compute/tasks/disk_migrate_task.go b/pkg/compute/tasks/disk_migrate_task.go index f11caa8c88..8efb7ca2d6 100644 --- a/pkg/compute/tasks/disk_migrate_task.go +++ b/pkg/compute/tasks/disk_migrate_task.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tasks import ( diff --git a/pkg/hostman/container/device/doc.go b/pkg/hostman/container/device/doc.go index e55bdb2fe9..f99e94a934 100644 --- a/pkg/hostman/container/device/doc.go +++ b/pkg/hostman/container/device/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package device // import "yunion.io/x/onecloud/pkg/hostman/container/device" diff --git a/pkg/hostman/container/device/host.go b/pkg/hostman/container/device/host.go index 5822da2bcc..afc27cbf79 100644 --- a/pkg/hostman/container/device/host.go +++ b/pkg/hostman/container/device/host.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package device import ( diff --git a/pkg/hostman/container/device/interface.go b/pkg/hostman/container/device/interface.go index 90fcf15a6c..d060f0d8e2 100644 --- a/pkg/hostman/container/device/interface.go +++ b/pkg/hostman/container/device/interface.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package device import ( diff --git a/pkg/hostman/container/device/isolated_device.go b/pkg/hostman/container/device/isolated_device.go index c080716572..ccccdd7e3e 100644 --- a/pkg/hostman/container/device/isolated_device.go +++ b/pkg/hostman/container/device/isolated_device.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package device import ( diff --git a/pkg/hostman/container/lifecycle/doc.go b/pkg/hostman/container/lifecycle/doc.go index 97042f4057..45dd7c0f36 100644 --- a/pkg/hostman/container/lifecycle/doc.go +++ b/pkg/hostman/container/lifecycle/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package lifecycle // import "yunion.io/x/onecloud/pkg/hostman/container/lifecycle" diff --git a/pkg/hostman/container/lifecycle/exec.go b/pkg/hostman/container/lifecycle/exec.go index 63928e74de..813efcd301 100644 --- a/pkg/hostman/container/lifecycle/exec.go +++ b/pkg/hostman/container/lifecycle/exec.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package lifecycle import ( diff --git a/pkg/hostman/container/lifecycle/lifecycle.go b/pkg/hostman/container/lifecycle/lifecycle.go index bfaf0cc8cd..a24daf7276 100644 --- a/pkg/hostman/container/lifecycle/lifecycle.go +++ b/pkg/hostman/container/lifecycle/lifecycle.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package lifecycle import ( diff --git a/pkg/hostman/container/storage/doc.go b/pkg/hostman/container/storage/doc.go index 0c3025babe..716196440c 100644 --- a/pkg/hostman/container/storage/doc.go +++ b/pkg/hostman/container/storage/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package storage // import "yunion.io/x/onecloud/pkg/hostman/container/storage" diff --git a/pkg/hostman/container/volume_mount/disk.go b/pkg/hostman/container/volume_mount/disk.go index a669304d16..98432c54ef 100644 --- a/pkg/hostman/container/volume_mount/disk.go +++ b/pkg/hostman/container/volume_mount/disk.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package volume_mount import ( diff --git a/pkg/hostman/container/volume_mount/doc.go b/pkg/hostman/container/volume_mount/doc.go index d7ed733763..da186b3583 100644 --- a/pkg/hostman/container/volume_mount/doc.go +++ b/pkg/hostman/container/volume_mount/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package volume_mount // import "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" diff --git a/pkg/hostman/container/volume_mount/host_path.go b/pkg/hostman/container/volume_mount/host_path.go index 04e209d1fe..061401b98f 100644 --- a/pkg/hostman/container/volume_mount/host_path.go +++ b/pkg/hostman/container/volume_mount/host_path.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package volume_mount import ( diff --git a/pkg/hostman/container/volume_mount/interface.go b/pkg/hostman/container/volume_mount/interface.go index 62a77858bf..1a7751b46b 100644 --- a/pkg/hostman/container/volume_mount/interface.go +++ b/pkg/hostman/container/volume_mount/interface.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package volume_mount import ( diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index be74bf17ab..cab72cdc2b 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -1007,13 +1007,8 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie } ctrCfg.Devices = append(ctrCfg.Devices, ctrDevs...) } - - nvMan, err := isolated_device.GetContainerDeviceManager(isolated_device.ContainerDeviceTypeNVIDIAGPU) - if err != nil { - return "", errors.Wrapf(err, "GetContainerDeviceManager by type %q", isolated_device.ContainerDeviceTypeNVIDIAGPU) - } - if envs := nvMan.GetContainerEnvs(spec.Devices); len(envs) > 0 { - ctrCfg.Envs = append(ctrCfg.Envs, envs...) + if err := s.getIsolatedDeviceExtraConfig(spec, ctrCfg); err != nil { + return "", err } } if len(spec.Command) != 0 { @@ -1032,6 +1027,24 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie return criId, nil } +func (s *sPodGuestInstance) getIsolatedDeviceExtraConfig(spec *hostapi.ContainerSpec, ctrCfg *runtimeapi.ContainerConfig) error { + devTypes := []isolated_device.ContainerDeviceType{isolated_device.ContainerDeviceTypeNvidiaGpu, isolated_device.ContainerDeviceTypeNvidiaMps} + for _, devType := range devTypes { + devMan, err := isolated_device.GetContainerDeviceManager(devType) + if err != nil { + return errors.Wrapf(err, "GetContainerDeviceManager by type %q", devType) + } + envs, mounts := devMan.GetContainerExtraConfigures(spec.Devices) + if len(envs) > 0 { + ctrCfg.Envs = append(ctrCfg.Envs, envs...) + } + if len(mounts) > 0 { + ctrCfg.Mounts = append(ctrCfg.Mounts, mounts...) + } + } + return nil +} + func (s *sPodGuestInstance) getContainerSystemCpusDir(ctrId string) string { return filepath.Join(s.HomeDir(), "cpus", ctrId) } diff --git a/pkg/hostman/guestman/podhandlers/doc.go b/pkg/hostman/guestman/podhandlers/doc.go index 940f288e05..497d9e483f 100644 --- a/pkg/hostman/guestman/podhandlers/doc.go +++ b/pkg/hostman/guestman/podhandlers/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package podhandlers // import "yunion.io/x/onecloud/pkg/hostman/guestman/podhandlers" diff --git a/pkg/hostman/guestman/podhandlers/podhandlers.go b/pkg/hostman/guestman/podhandlers/podhandlers.go index c29ab3ea94..653cea8325 100644 --- a/pkg/hostman/guestman/podhandlers/podhandlers.go +++ b/pkg/hostman/guestman/podhandlers/podhandlers.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package podhandlers import ( diff --git a/pkg/hostman/guestman/runtime.go b/pkg/hostman/guestman/runtime.go index 8413273ccc..779c15a687 100644 --- a/pkg/hostman/guestman/runtime.go +++ b/pkg/hostman/guestman/runtime.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package guestman import ( diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 5c3b34c2ad..80b7cd98e7 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2134,9 +2134,16 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { return nil, err } h.IsolatedDeviceMan.ProbePCIDevices( - options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, options.HostOptions.DisableCustomDevice, - sriovNics, offloadNics, options.HostOptions.PTNVMEConfigs, options.HostOptions.AMDVgpuPFs, options.HostOptions.NVIDIAVgpuPFs, - enableDevWhitelist) + options.HostOptions.DisableGPU, + options.HostOptions.DisableUSB, + options.HostOptions.DisableCustomDevice, + sriovNics, offloadNics, + options.HostOptions.PTNVMEConfigs, + options.HostOptions.AMDVgpuPFs, + options.HostOptions.NVIDIAVgpuPFs, + options.HostOptions.EnableCudaMPS, + enableDevWhitelist, + ) objs, err := h.getRemoteIsolatedDevices() if err != nil { diff --git a/pkg/hostman/isolated_device/container_device.go b/pkg/hostman/isolated_device/container_device.go index 00333d3abe..e1be9e26cd 100644 --- a/pkg/hostman/isolated_device/container_device.go +++ b/pkg/hostman/isolated_device/container_device.go @@ -36,7 +36,8 @@ const ( ContainerDeviceTypeCphASOPBinder ContainerDeviceType = api.CONTAINER_DEV_CPH_AOSP_BINDER ContainerNetintCAASIC ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_ASIC ContainerNetintCAQuadra ContainerDeviceType = api.CONTAINER_DEV_NETINT_CA_QUADRA - ContainerDeviceTypeNVIDIAGPU ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU + ContainerDeviceTypeNvidiaGpu ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU + ContainerDeviceTypeNvidiaMps ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS ) func GetContainerDeviceManager(t ContainerDeviceType) (IContainerDeviceManager, error) { @@ -69,5 +70,5 @@ type IContainerDeviceManager interface { NewDevices(dev *ContainerDevice) ([]IDevice, error) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) ProbeDevices() ([]IDevice, error) - GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue + GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) } diff --git a/pkg/hostman/isolated_device/container_device/base_dev.go b/pkg/hostman/isolated_device/container_device/base_dev.go index 18a12846d0..763c7ba1b3 100644 --- a/pkg/hostman/isolated_device/container_device/base_dev.go +++ b/pkg/hostman/isolated_device/container_device/base_dev.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package container_device import ( @@ -38,6 +52,18 @@ func (c BaseDevice) GetDevicePath() string { return c.Path } +func (c BaseDevice) GetNvidiaMpsMemoryLimit() int { + return -1 +} + +func (c BaseDevice) GetNvidiaMpsMemoryTotal() int { + return -1 +} + +func (c BaseDevice) GetNvidiaMpsThreadPercentage() int { + return -1 +} + func CheckVirtualNumber(dev *isolated_device.ContainerDevice) error { if dev.VirtualNumber <= 0 { return errors.Errorf("virtual_number must > 0") diff --git a/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go b/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go index 61b1900aa3..7bc2253081 100644 --- a/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go +++ b/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go @@ -81,8 +81,8 @@ func (m *cphAMDGPUManager) NewContainerDevices(_ *hostapi.ContainerCreateInput, return []*runtimeapi.Device{cDev}, nil } -func (m *cphAMDGPUManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue { - return nil +func (m *cphAMDGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + return nil, nil } type cphAMDGPU struct { diff --git a/pkg/hostman/isolated_device/container_device/cph_amd_gpu_test.go b/pkg/hostman/isolated_device/container_device/cph_amd_gpu_test.go index bb976a1ed9..90d79d1b67 100644 --- a/pkg/hostman/isolated_device/container_device/cph_amd_gpu_test.go +++ b/pkg/hostman/isolated_device/container_device/cph_amd_gpu_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package container_device import "testing" diff --git a/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go b/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go index aa024d8409..56be083b17 100644 --- a/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go +++ b/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go @@ -123,8 +123,8 @@ func (m *cphAOSPBinderManager) NewContainerDevices(ctrInput *hostapi.ContainerCr return ctrDevs, nil } -func (m *cphAOSPBinderManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue { - return nil +func (m *cphAOSPBinderManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + return nil, nil } func (m *cphAOSPBinderManager) ensureBinderDeviceOldWay(dev *hostapi.ContainerIsolatedDevice) error { diff --git a/pkg/hostman/isolated_device/container_device/doc.go b/pkg/hostman/isolated_device/container_device/doc.go index 6ed08228a8..2d38138799 100644 --- a/pkg/hostman/isolated_device/container_device/doc.go +++ b/pkg/hostman/isolated_device/container_device/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package container_device // import "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device" diff --git a/pkg/hostman/isolated_device/container_device/net_int_device.go b/pkg/hostman/isolated_device/container_device/net_int_device.go index 5d5644af33..02eaa45bde 100644 --- a/pkg/hostman/isolated_device/container_device/net_int_device.go +++ b/pkg/hostman/isolated_device/container_device/net_int_device.go @@ -166,8 +166,8 @@ func (m *netintDeviceManager) NewContainerDevices(_ *hostapi.ContainerCreateInpu return ctrDevs, nil } -func (m *netintDeviceManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue { - return nil +func (m *netintDeviceManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + return nil, nil } type netintDevice struct { diff --git a/pkg/hostman/isolated_device/container_device/nvidia_gpu.go b/pkg/hostman/isolated_device/container_device/nvidia_gpu.go index e44f7a07e8..a4fa1feecd 100644 --- a/pkg/hostman/isolated_device/container_device/nvidia_gpu.go +++ b/pkg/hostman/isolated_device/container_device/nvidia_gpu.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package container_device import ( @@ -8,7 +22,6 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/onecloud/pkg/apis/host" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/util/procutils" @@ -25,7 +38,7 @@ func newNvidiaGPUManager() *nvidiaGPUManager { } func (m *nvidiaGPUManager) GetType() isolated_device.ContainerDeviceType { - return isolated_device.ContainerDeviceTypeNVIDIAGPU + return isolated_device.ContainerDeviceTypeNvidiaGpu } func (m *nvidiaGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { @@ -40,19 +53,19 @@ func (m *nvidiaGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInp return nil, nil } -func (m *nvidiaGPUManager) GetContainerEnvs(devs []*host.ContainerDevice) []*runtimeapi.KeyValue { +func (m *nvidiaGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { gpuIds := []string{} for _, dev := range devs { if dev.IsolatedDevice == nil { continue } - if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNVIDIAGPU { + if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNvidiaGpu { continue } gpuIds = append(gpuIds, dev.IsolatedDevice.Path) } if len(gpuIds) == 0 { - return nil + return nil, nil } return []*runtimeapi.KeyValue{ @@ -64,7 +77,7 @@ func (m *nvidiaGPUManager) GetContainerEnvs(devs []*host.ContainerDevice) []*run Key: "NVIDIA_DRIVER_CAPABILITIES", Value: "all", }, - } + }, nil } type nvidiaGPU struct { @@ -76,7 +89,9 @@ func getNvidiaGPUs() ([]isolated_device.IDevice, error) { // nvidia-smi --query-gpu=gpu_uuid,gpu_name,gpu_bus_id --format=csv // uuid, name, pci.bus_id // GPU-bc1a3bb9-55cb-8c52-c374-4f8b4f388a20, NVIDIA A800-SXM4-80GB, 00000000:10:00.0 - out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id", "--format=csv").Output() + + // nvidia-smi --query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode --format=csv + out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id,compute_mode", "--format=csv").Output() if err != nil { return nil, errors.Wrap(err, "nvidia-smi") } @@ -90,14 +105,19 @@ func getNvidiaGPUs() ([]isolated_device.IDevice, error) { log.Errorf("unknown nvidia-smi out line %s", line) continue } - gpuId, gpuName, gpuPciAddr := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]) + gpuId, gpuName, gpuPciAddr, computeMode := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]), strings.TrimSpace(segs[3]) + if computeMode != "Default" { + log.Warningf("gpu device %s compute mode %s, skip.", gpuId, computeMode) + continue + } + pciOutput, err := isolated_device.GetPCIStrByAddr(gpuPciAddr) if err != nil { return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", gpuPciAddr) } dev := isolated_device.NewPCIDevice2(pciOutput[0]) gpuDev := &nvidiaGPU{ - BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNVIDIAGPU, gpuId), + BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNvidiaGpu, gpuId), } gpuDev.SetModelName(gpuName) diff --git a/pkg/hostman/isolated_device/container_device/nvidia_mps.go b/pkg/hostman/isolated_device/container_device/nvidia_mps.go new file mode 100644 index 0000000000..47157fde87 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/nvidia_mps.go @@ -0,0 +1,187 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "fmt" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +// The MPS /dev/shm is needed to allow MPS daemon health-checking +var shmPath = "/dev/shm" + +func init() { + isolated_device.RegisterContainerDeviceManager(newNvidiaMPSManager()) +} + +type nvidiaMPSManager struct{} + +func newNvidiaMPSManager() *nvidiaMPSManager { + return &nvidiaMPSManager{} +} + +func (m *nvidiaMPSManager) GetType() isolated_device.ContainerDeviceType { + return isolated_device.ContainerDeviceTypeNvidiaMps +} + +func (m *nvidiaMPSManager) ProbeDevices() ([]isolated_device.IDevice, error) { + return getNvidiaMPSGpus() +} + +func (m *nvidiaMPSManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { + return nil, nil +} + +func (m *nvidiaMPSManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, error) { + return nil, nil +} + +func (m *nvidiaMPSManager) getMPSPipeDirectory() string { + return options.HostOptions.CudaMPSPipeDirectory +} + +func (m *nvidiaMPSManager) getSHMPath() string { + return shmPath +} + +func (m *nvidiaMPSManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + gpuIds := []string{} + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNvidiaMps { + continue + } + gpuIds = append(gpuIds, dev.IsolatedDevice.Path) + } + if len(gpuIds) == 0 { + return nil, nil + } + + return []*runtimeapi.KeyValue{ + { + Key: "CUDA_MPS_PIPE_DIRECTORY", + Value: m.getMPSPipeDirectory(), + }, + { + Key: "NVIDIA_VISIBLE_DEVICES", + Value: strings.Join(gpuIds, ","), + }, + { + Key: "NVIDIA_DRIVER_CAPABILITIES", + Value: "all", + }, + }, []*runtimeapi.Mount{ + { + ContainerPath: m.getSHMPath(), + HostPath: m.getSHMPath(), + }, + { + ContainerPath: m.getMPSPipeDirectory(), + HostPath: m.getMPSPipeDirectory(), + }, + } +} + +type nvidiaMPS struct { + *BaseDevice + + MemSizeMB int + MemTotalMB int + ThreadPercentage int +} + +func (c *nvidiaMPS) GetNvidiaMpsMemoryLimit() int { + return c.MemSizeMB +} + +func (c *nvidiaMPS) GetNvidiaMpsMemoryTotal() int { + return c.MemTotalMB +} + +func (c *nvidiaMPS) GetNvidiaMpsThreadPercentage() int { + return c.ThreadPercentage +} + +func parseMemSize(memTotalStr string) (int, error) { + if !strings.HasSuffix(memTotalStr, " MiB") { + return -1, errors.Errorf("unknown mem string suffix") + } + memStr := strings.TrimSpace(strings.TrimSuffix(memTotalStr, " MiB")) + return strconv.Atoi(memStr) +} + +func getNvidiaMPSGpus() ([]isolated_device.IDevice, error) { + devs := make([]isolated_device.IDevice, 0) + // nvidia-smi --query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode --format=csv + // GPU-76aef7ff-372d-2432-b4b4-beca4d8d3400, Tesla P40, 00000000:00:08.0, 23040 MiB, Exclusive_Process + out, err := procutils.NewRemoteCommandAsFarAsPossible("nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,gpu_bus_id,memory.total,compute_mode", "--format=csv").Output() + if err != nil { + return nil, errors.Wrap(err, "nvidia-smi") + } + lines := strings.Split(string(out), "\n") + for _, line := range lines { + if strings.HasPrefix(line, "uuid") { + continue + } + segs := strings.Split(line, ",") + if len(segs) != 5 { + log.Errorf("unknown nvidia-smi out line %s", line) + continue + } + gpuId, gpuName, gpuPciAddr, memTotal, computeMode := strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]), strings.TrimSpace(segs[2]), strings.TrimSpace(segs[3]), strings.TrimSpace(segs[4]) + if computeMode != "Exclusive_Process" { + log.Warningf("gpu device %s compute mode %s, skip.", gpuId, computeMode) + continue + } + memSize, err := parseMemSize(memTotal) + if err != nil { + return nil, errors.Wrapf(err, "failed parse memSize %s", memTotal) + } + + pciOutput, err := isolated_device.GetPCIStrByAddr(gpuPciAddr) + if err != nil { + return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", gpuPciAddr) + } + for i := 0; i < options.HostOptions.CudaMPSReplicas; i++ { + dev := isolated_device.NewPCIDevice2(pciOutput[0]) + gpuDev := &nvidiaMPS{ + BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNvidiaMps, gpuId), + MemSizeMB: memSize / options.HostOptions.CudaMPSReplicas, + MemTotalMB: memSize, + ThreadPercentage: 100 / options.HostOptions.CudaMPSReplicas, + } + gpuDev.SetModelName(gpuName) + gpuDev.SetAddr(fmt.Sprintf("%s-%d", gpuDev.GetAddr(), i)) + devs = append(devs, gpuDev) + } + } + if len(devs) == 0 { + return nil, nil + } + return devs, nil +} diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 10e3f560a3..9869ad29b5 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -111,13 +111,18 @@ type IDevice interface { // Get extra PCIE information GetPCIEInfo() *api.IsolatedDevicePCIEInfo GetDevicePath() string + + // mps infos + GetNvidiaMpsMemoryLimit() int + GetNvidiaMpsMemoryTotal() int + GetNvidiaMpsThreadPercentage() int } type IsolatedDeviceManager interface { GetDevices() []IDevice GetDeviceByIdent(vendorDevId, addr, mdevId string) IDevice GetDeviceByAddr(addr string) IDevice - ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool) + ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableCudaMps, enableWhitelist bool) StartDetachTask() BatchCustomProbe() AppendDetachedDevice(dev *CloudDeviceInfo) @@ -188,10 +193,15 @@ func (man *isolatedDeviceManager) probeContainerDevices() { } } -func (man *isolatedDeviceManager) probeContainerNvidiaGPUs() { - devman, err := GetContainerDeviceManager(ContainerDeviceTypeNVIDIAGPU) +func (man *isolatedDeviceManager) probeContainerNvidiaGPUs(enableCudaMps bool) { + devType := ContainerDeviceTypeNvidiaGpu + if enableCudaMps { + devType = ContainerDeviceTypeNvidiaMps + } + + devman, err := GetContainerDeviceManager(devType) if err != nil { - log.Errorf("no container device manager %s found", ContainerDeviceTypeNVIDIAGPU) + log.Errorf("no container device manager %s found", devType) return } devs, err := devman.ProbeDevices() @@ -369,10 +379,15 @@ func (man *isolatedDeviceManager) probeNVIDIAVgpus(nvidiaVgpuPFs []string) { } } -func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool) { +func (man *isolatedDeviceManager) ProbePCIDevices( + skipGPUs, skipUSBs, skipCustomDevs bool, + sriovNics, ovsOffloadNics []HostNic, + nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, + enableCudaMps, enableWhitelist bool, +) { man.devices = make([]IDevice, 0) if man.host.IsContainerHost() { - man.probeContainerNvidiaGPUs() + man.probeContainerNvidiaGPUs(enableCudaMps) man.probeContainerDevices() } else { devModels, err := man.getCustomIsolatedDeviceModels() @@ -569,6 +584,10 @@ func (dev *SBaseDevice) GetAddr() string { return dev.dev.Addr } +func (dev *SBaseDevice) SetAddr(addr string) { + dev.dev.Addr = addr +} + func (dev *SBaseDevice) GetDeviceType() string { return dev.devType } @@ -628,6 +647,18 @@ func (dev *SBaseDevice) GetGuestId() string { return dev.guestId } +func (dev *SBaseDevice) GetNvidiaMpsMemoryLimit() int { + return -1 +} + +func (dev *SBaseDevice) GetNvidiaMpsMemoryTotal() int { + return -1 +} + +func (dev *SBaseDevice) GetNvidiaMpsThreadPercentage() int { + return -1 +} + func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { data := map[string]interface{}{ "dev_type": dev.GetDeviceType(), @@ -682,6 +713,16 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { if devPath != "" { data["device_path"] = devPath } + + if mpsMemTotal := dev.GetNvidiaMpsMemoryTotal(); mpsMemTotal > 0 { + data["mps_memory_total"] = mpsMemTotal + } + if mpsMemLimit := dev.GetNvidiaMpsMemoryLimit(); mpsMemLimit > 0 { + data["mps_memory_limit"] = mpsMemLimit + } + if mpsThreadPercentage := dev.GetNvidiaMpsThreadPercentage(); mpsThreadPercentage > 0 { + data["mps_thread_percentage"] = mpsThreadPercentage + } return jsonutils.Marshal(data).(*jsonutils.JSONDict) } diff --git a/pkg/hostman/isolated_device/nvidia_vgpu.go b/pkg/hostman/isolated_device/nvidia_vgpu.go index ec711c642f..29ad5fed69 100644 --- a/pkg/hostman/isolated_device/nvidia_vgpu.go +++ b/pkg/hostman/isolated_device/nvidia_vgpu.go @@ -101,6 +101,18 @@ func (dev *sNVIDIAVgpuDevice) GetDevicePath() string { return "" } +func (dev *sNVIDIAVgpuDevice) GetNvidiaMpsMemoryLimit() int { + return -1 +} + +func (dev *sNVIDIAVgpuDevice) GetNvidiaMpsMemoryTotal() int { + return -1 +} + +func (dev *sNVIDIAVgpuDevice) GetNvidiaMpsThreadPercentage() int { + return -1 +} + func (dev *sNVIDIAVgpuDevice) SetDeviceInfo(info CloudDeviceInfo) { if len(info.Id) != 0 { dev.cloudId = info.Id diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 30345c82fe..e5e4e1d40a 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -212,6 +212,11 @@ type SHostOptions struct { ContainerRuntimeEndpoint string `help:"endpoint of container runtime service" default:"unix:///var/run/onecloud/containerd/containerd.sock"` ContainerDeviceConfigFile string `help:"container device configuration file path"` LxcfsPath string `help:"lxcfs directory path" default:"/var/lib/lxcfs"` + + EnableCudaMPS bool `help:"enable cuda mps" default:"false"` + CudaMPSPipeDirectory string `help:"cuda mps pipe dir" default:"/tmp/nvidia-mps/pipe"` + CudaMPSLogDirectory string `help:"cuda mps log dir" default:"/tmp/nvidia-mps/log"` + CudaMPSReplicas int `help:"cuda mps replias" default:"10"` } var ( diff --git a/pkg/hostman/storageman/container_storage/doc.go b/pkg/hostman/storageman/container_storage/doc.go index af93c57a13..c38c027ae5 100644 --- a/pkg/hostman/storageman/container_storage/doc.go +++ b/pkg/hostman/storageman/container_storage/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package container_storage // import "yunion.io/x/onecloud/pkg/hostman/storageman/container_storage" diff --git a/pkg/hostman/storageman/container_storage/local_raw.go b/pkg/hostman/storageman/container_storage/local_raw.go index 89cdb6d026..b5c0c52d0d 100644 --- a/pkg/hostman/storageman/container_storage/local_raw.go +++ b/pkg/hostman/storageman/container_storage/local_raw.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package container_storage import ( @@ -57,8 +71,8 @@ func (m *localLoopDiskManager) ProbeDevices() ([]isolated_device.IDevice, error) return nil, nil } -func (m *localLoopDiskManager) GetContainerEnvs(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue { - return nil +func (m *localLoopDiskManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + return nil, nil } func newLocalLoopDiskManager() *localLoopDiskManager { diff --git a/pkg/mcclient/modules/compute/mod_host_isolated_device_models.go b/pkg/mcclient/modules/compute/mod_host_isolated_device_models.go index 5d29f62e06..00b2f2aeb1 100644 --- a/pkg/mcclient/modules/compute/mod_host_isolated_device_models.go +++ b/pkg/mcclient/modules/compute/mod_host_isolated_device_models.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package compute import ( diff --git a/pkg/mcclient/options/compute/server_pod_test.go b/pkg/mcclient/options/compute/server_pod_test.go index c145c52447..2c172d5ae1 100644 --- a/pkg/mcclient/options/compute/server_pod_test.go +++ b/pkg/mcclient/options/compute/server_pod_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package compute import ( diff --git a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go index b5642414a5..f83154f813 100644 --- a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go +++ b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go @@ -56,6 +56,19 @@ func (f *IsolatedDevicePredicate) PreExecute(ctx context.Context, u *core.Unit, return false, nil } +func (f *IsolatedDevicePredicate) getIsolatedDeviceCountByType(getter core.CandidatePropertyGetter, devType string) int { + devs := getter.UnusedIsolatedDevicesByType(devType) + if devType != compute.CONTAINER_DEV_NVIDIA_MPS { + return len(devs) + } else { + devMap := map[string]struct{}{} + for _, dev := range devs { + devMap[dev.DevicePath] = struct{}{} + } + return len(devMap) + } +} + func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := NewPredicateHelper(f, u, c) reqIsoDevs := u.SchedData().IsolatedDevices @@ -116,7 +129,7 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c } } for devType, reqCount := range devTypeRequest { - freeCount := len(getter.UnusedIsolatedDevicesByType(devType)) + freeCount := f.getIsolatedDeviceCountByType(getter, devType) if freeCount < reqCount { h.Exclude(fmt.Sprintf("IsolatedDevice type %q not enough, request: %d, hostFree: %d", devType, reqCount, freeCount)) return h.GetResult() diff --git a/pkg/scheduler/data_manager/common/cache_manager.go b/pkg/scheduler/data_manager/common/cache_manager.go index b842ee60ce..cc5af80abe 100644 --- a/pkg/scheduler/data_manager/common/cache_manager.go +++ b/pkg/scheduler/data_manager/common/cache_manager.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package common var cacheManager CacheManager diff --git a/pkg/util/httpstream/doc.go b/pkg/util/httpstream/doc.go index fff4f9cf56..8a83769260 100644 --- a/pkg/util/httpstream/doc.go +++ b/pkg/util/httpstream/doc.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/httpstream/httpstream.go b/pkg/util/httpstream/httpstream.go index 62f2e84e50..6a893408da 100644 --- a/pkg/util/httpstream/httpstream.go +++ b/pkg/util/httpstream/httpstream.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/httpstream/httpstream_test.go b/pkg/util/httpstream/httpstream_test.go index f7f9a3ebf4..f324877fe3 100644 --- a/pkg/util/httpstream/httpstream_test.go +++ b/pkg/util/httpstream/httpstream_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/httpstream/spdy/connection.go b/pkg/util/httpstream/spdy/connection.go index 5eb35daa40..7f78734c9a 100644 --- a/pkg/util/httpstream/spdy/connection.go +++ b/pkg/util/httpstream/spdy/connection.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/httpstream/spdy/connection_test.go b/pkg/util/httpstream/spdy/connection_test.go index 112ed991f7..95581cd533 100644 --- a/pkg/util/httpstream/spdy/connection_test.go +++ b/pkg/util/httpstream/spdy/connection_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/httpstream/spdy/doc.go b/pkg/util/httpstream/spdy/doc.go index 45d19ef212..98d3a31f9f 100644 --- a/pkg/util/httpstream/spdy/doc.go +++ b/pkg/util/httpstream/spdy/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package spdy // import "yunion.io/x/onecloud/pkg/util/httpstream/spdy" diff --git a/pkg/util/httpstream/spdy/roundtripper.go b/pkg/util/httpstream/spdy/roundtripper.go index c45897c908..a4c591ce66 100644 --- a/pkg/util/httpstream/spdy/roundtripper.go +++ b/pkg/util/httpstream/spdy/roundtripper.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/httpstream/spdy/roundtripper_test.go b/pkg/util/httpstream/spdy/roundtripper_test.go index 201fe00061..490f867ca1 100644 --- a/pkg/util/httpstream/spdy/roundtripper_test.go +++ b/pkg/util/httpstream/spdy/roundtripper_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/httpstream/spdy/upgrade.go b/pkg/util/httpstream/spdy/upgrade.go index 936ac624df..b8cc15716a 100644 --- a/pkg/util/httpstream/spdy/upgrade.go +++ b/pkg/util/httpstream/spdy/upgrade.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/httpstream/spdy/upgrade_test.go b/pkg/util/httpstream/spdy/upgrade_test.go index 5a514dd5bf..de51997996 100644 --- a/pkg/util/httpstream/spdy/upgrade_test.go +++ b/pkg/util/httpstream/spdy/upgrade_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/interrupt/doc.go b/pkg/util/interrupt/doc.go index 7d2edae14f..18d35c1eee 100644 --- a/pkg/util/interrupt/doc.go +++ b/pkg/util/interrupt/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package interrupt // import "yunion.io/x/onecloud/pkg/util/interrupt" diff --git a/pkg/util/interrupt/interrupt.go b/pkg/util/interrupt/interrupt.go index 0265b9fb17..d76943aa96 100644 --- a/pkg/util/interrupt/interrupt.go +++ b/pkg/util/interrupt/interrupt.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/losetup/device.go b/pkg/util/losetup/device.go index 667e4b246c..91cb97ae9e 100644 --- a/pkg/util/losetup/device.go +++ b/pkg/util/losetup/device.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package losetup import ( diff --git a/pkg/util/losetup/device_test.go b/pkg/util/losetup/device_test.go index 8eeac198ee..c860691fc3 100644 --- a/pkg/util/losetup/device_test.go +++ b/pkg/util/losetup/device_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package losetup import ( diff --git a/pkg/util/losetup/doc.go b/pkg/util/losetup/doc.go index 8677eaf604..c696f27b1a 100644 --- a/pkg/util/losetup/doc.go +++ b/pkg/util/losetup/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package losetup // import "yunion.io/x/onecloud/pkg/util/losetup" diff --git a/pkg/util/losetup/losetup.go b/pkg/util/losetup/losetup.go index f5bb5a8a81..9ea1d088d9 100644 --- a/pkg/util/losetup/losetup.go +++ b/pkg/util/losetup/losetup.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package losetup import ( diff --git a/pkg/util/netutils2/getport/doc.go b/pkg/util/netutils2/getport/doc.go index 7b07bcbdd9..9fec5bf26c 100644 --- a/pkg/util/netutils2/getport/doc.go +++ b/pkg/util/netutils2/getport/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package getport // import "yunion.io/x/onecloud/pkg/util/netutils2/getport" diff --git a/pkg/util/netutils2/getport/getport_test.go b/pkg/util/netutils2/getport/getport_test.go index 54584143cf..c2e6882cab 100644 --- a/pkg/util/netutils2/getport/getport_test.go +++ b/pkg/util/netutils2/getport/getport_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package getport import ( diff --git a/pkg/util/pod/cpus.go b/pkg/util/pod/cpus.go index f69197e6d0..7aa2afe9d9 100644 --- a/pkg/util/pod/cpus.go +++ b/pkg/util/pod/cpus.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pod import ( diff --git a/pkg/util/pod/doc.go b/pkg/util/pod/doc.go index a686d0c730..1ab83d89c2 100644 --- a/pkg/util/pod/doc.go +++ b/pkg/util/pod/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pod // import "yunion.io/x/onecloud/pkg/util/pod" diff --git a/pkg/util/pod/pod.go b/pkg/util/pod/pod.go index 8212f4ab54..739ba3829a 100644 --- a/pkg/util/pod/pod.go +++ b/pkg/util/pod/pod.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pod import ( diff --git a/pkg/util/pod/remotecommand/constants.go b/pkg/util/pod/remotecommand/constants.go index acfeb827c9..9560f13660 100644 --- a/pkg/util/pod/remotecommand/constants.go +++ b/pkg/util/pod/remotecommand/constants.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/pod/remotecommand/doc.go b/pkg/util/pod/remotecommand/doc.go index ad2d2e64ce..79969ad5d1 100644 --- a/pkg/util/pod/remotecommand/doc.go +++ b/pkg/util/pod/remotecommand/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package remotecommand // import "yunion.io/x/onecloud/pkg/util/pod/remotecommand" diff --git a/pkg/util/pod/remotecommand/errorstream.go b/pkg/util/pod/remotecommand/errorstream.go index 4844e928a3..167f95832c 100644 --- a/pkg/util/pod/remotecommand/errorstream.go +++ b/pkg/util/pod/remotecommand/errorstream.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/pod/remotecommand/executor.go b/pkg/util/pod/remotecommand/executor.go index a98f7360b3..8b2d56149e 100644 --- a/pkg/util/pod/remotecommand/executor.go +++ b/pkg/util/pod/remotecommand/executor.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package remotecommand import ( diff --git a/pkg/util/pod/remotecommand/reader.go b/pkg/util/pod/remotecommand/reader.go index c3de88c733..dfbe2d6579 100644 --- a/pkg/util/pod/remotecommand/reader.go +++ b/pkg/util/pod/remotecommand/reader.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package remotecommand import ( diff --git a/pkg/util/pod/remotecommand/remotecommand.go b/pkg/util/pod/remotecommand/remotecommand.go index cdd2067f43..72d53c6f32 100644 --- a/pkg/util/pod/remotecommand/remotecommand.go +++ b/pkg/util/pod/remotecommand/remotecommand.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package remotecommand import ( diff --git a/pkg/util/pod/remotecommand/resize.go b/pkg/util/pod/remotecommand/resize.go index eb7177a616..5357aac2f1 100644 --- a/pkg/util/pod/remotecommand/resize.go +++ b/pkg/util/pod/remotecommand/resize.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package remotecommand // TerminalSize and TerminalSizeQueue was a port of k8s.io/kubernetes/pkg/util/term diff --git a/pkg/util/pod/remotecommand/spdy/doc.go b/pkg/util/pod/remotecommand/spdy/doc.go index cec450e6fd..132b0bfa03 100644 --- a/pkg/util/pod/remotecommand/spdy/doc.go +++ b/pkg/util/pod/remotecommand/spdy/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package spdy // import "yunion.io/x/onecloud/pkg/util/pod/remotecommand/spdy" diff --git a/pkg/util/pod/remotecommand/spdy/spdy.go b/pkg/util/pod/remotecommand/spdy/spdy.go index 6a875500c6..3f745e7bc5 100644 --- a/pkg/util/pod/remotecommand/spdy/spdy.go +++ b/pkg/util/pod/remotecommand/spdy/spdy.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package spdy import ( diff --git a/pkg/util/pod/remotecommand/transport.go b/pkg/util/pod/remotecommand/transport.go index 9dd256e6d5..6b30e804cc 100644 --- a/pkg/util/pod/remotecommand/transport.go +++ b/pkg/util/pod/remotecommand/transport.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package remotecommand diff --git a/pkg/util/pod/remotecommand/v1.go b/pkg/util/pod/remotecommand/v1.go index 49692e83f8..a7e950f3a5 100644 --- a/pkg/util/pod/remotecommand/v1.go +++ b/pkg/util/pod/remotecommand/v1.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/pod/remotecommand/v2.go b/pkg/util/pod/remotecommand/v2.go index a58fb4ecca..b4142e8c5d 100644 --- a/pkg/util/pod/remotecommand/v2.go +++ b/pkg/util/pod/remotecommand/v2.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/util/pod/remotecommand/v2_test.go b/pkg/util/pod/remotecommand/v2_test.go index 6f0aebc062..8db19bd254 100644 --- a/pkg/util/pod/remotecommand/v2_test.go +++ b/pkg/util/pod/remotecommand/v2_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/pod/remotecommand/v3.go b/pkg/util/pod/remotecommand/v3.go index 39e7c0737e..65f5b8307f 100644 --- a/pkg/util/pod/remotecommand/v3.go +++ b/pkg/util/pod/remotecommand/v3.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/pod/remotecommand/v4.go b/pkg/util/pod/remotecommand/v4.go index c3573c155a..c14b881307 100644 --- a/pkg/util/pod/remotecommand/v4.go +++ b/pkg/util/pod/remotecommand/v4.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/pod/term/doc.go b/pkg/util/pod/term/doc.go index 9c84c6ccf9..080503a7f1 100644 --- a/pkg/util/pod/term/doc.go +++ b/pkg/util/pod/term/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package term // import "yunion.io/x/onecloud/pkg/util/pod/term" diff --git a/pkg/util/pod/term/resize.go b/pkg/util/pod/term/resize.go index d8add0c15e..eb5cd0d997 100644 --- a/pkg/util/pod/term/resize.go +++ b/pkg/util/pod/term/resize.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/pod/term/resizeevents.go b/pkg/util/pod/term/resizeevents.go index 27653d269c..722e00ed85 100644 --- a/pkg/util/pod/term/resizeevents.go +++ b/pkg/util/pod/term/resizeevents.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build !windows // +build !windows diff --git a/pkg/util/pod/term/resizeevents_windows.go b/pkg/util/pod/term/resizeevents_windows.go index 162c9e1b60..134c49daa3 100644 --- a/pkg/util/pod/term/resizeevents_windows.go +++ b/pkg/util/pod/term/resizeevents_windows.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/pod/term/term.go b/pkg/util/pod/term/term.go index fa22603a57..9b275e6c83 100644 --- a/pkg/util/pod/term/term.go +++ b/pkg/util/pod/term/term.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package term import ( diff --git a/pkg/util/pod/term/term_writer.go b/pkg/util/pod/term/term_writer.go index 7f51b46685..7b43d55f87 100644 --- a/pkg/util/pod/term/term_writer.go +++ b/pkg/util/pod/term/term_writer.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors. diff --git a/pkg/util/pod/term/term_writer_test.go b/pkg/util/pod/term/term_writer_test.go index 7d22d6c888..2a6239c9ab 100644 --- a/pkg/util/pod/term/term_writer_test.go +++ b/pkg/util/pod/term/term_writer_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + /* Copyright 2016 The Kubernetes Authors.