From b5d8522f3797663f73f653a531d026a82fa5b3b4 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 23 Feb 2022 16:07:01 +0800 Subject: [PATCH] feat(region,host): server cpuset supported (#13527) * feat(region,host): server cpuset supported * feat(region,host): cpuset remove --- cmd/climc/shell/compute/hosts.go | 28 ++ cmd/climc/shell/compute/servers.go | 2 + go.mod | 3 +- go.sum | 14 +- pkg/apis/compute/guest_const.go | 1 + pkg/apis/compute/guests.go | 14 + pkg/apis/host/types.go | 13 + pkg/cloudcommon/db/opslog_const.go | 5 + pkg/compute/guestdrivers/base.go | 8 + pkg/compute/guestdrivers/kvm.go | 31 +++ pkg/compute/models/guest_actions.go | 67 +++++ pkg/compute/models/guestdrivers.go | 3 + pkg/compute/tasks/guest_cpuset_task.go | 67 +++++ .../guestman/guesthandlers/guesthandler.go | 19 ++ pkg/hostman/guestman/guestman.go | 16 ++ pkg/hostman/guestman/qemu-kvm.go | 61 ++++ pkg/hostman/hostinfo/hostinfo.go | 7 + pkg/hostman/hostinfo/hostinfohelper.go | 3 + pkg/hostman/hostutils/hardware/cpu.go | 31 +++ pkg/hostman/hostutils/hardware/doc.go | 1 + pkg/hostman/hostutils/hardware/topology.go | 31 +++ pkg/mcclient/options/compute/servers.go | 25 ++ pkg/util/logclient/consts.go | 2 + .../StackExchange/wmi/swbemservices.go | 2 +- vendor/github.com/StackExchange/wmi/wmi.go | 17 +- vendor/github.com/go-ole/go-ole/.travis.yml | 7 +- vendor/github.com/go-ole/go-ole/go.mod | 2 - vendor/github.com/go-ole/go-ole/go.sum | 2 - vendor/github.com/jaypipes/ghw/COPYING | 176 ++++++++++++ .../jaypipes/ghw/pkg/context/context.go | 128 +++++++++ vendor/github.com/jaypipes/ghw/pkg/cpu/cpu.go | 169 +++++++++++ .../jaypipes/ghw/pkg/cpu/cpu_linux.go | 220 +++++++++++++++ .../jaypipes/ghw/pkg/cpu/cpu_stub.go | 17 ++ .../jaypipes/ghw/pkg/cpu/cpu_windows.go | 55 ++++ .../jaypipes/ghw/pkg/linuxpath/path_linux.go | 71 +++++ .../jaypipes/ghw/pkg/marshal/marshal.go | 47 ++++ .../jaypipes/ghw/pkg/memory/memory.go | 80 ++++++ .../jaypipes/ghw/pkg/memory/memory_cache.go | 101 +++++++ .../ghw/pkg/memory/memory_cache_linux.go | 188 +++++++++++++ .../jaypipes/ghw/pkg/memory/memory_linux.go | 237 ++++++++++++++++ .../jaypipes/ghw/pkg/memory/memory_stub.go | 17 ++ .../jaypipes/ghw/pkg/memory/memory_windows.go | 72 +++++ .../jaypipes/ghw/pkg/option/option.go | 227 +++++++++++++++ .../jaypipes/ghw/pkg/pci/address/address.go | 55 ++++ .../jaypipes/ghw/pkg/snapshot/clonetree.go | 263 ++++++++++++++++++ .../ghw/pkg/snapshot/clonetree_block.go | 221 +++++++++++++++ .../ghw/pkg/snapshot/clonetree_gpu.go | 33 +++ .../ghw/pkg/snapshot/clonetree_net.go | 31 +++ .../ghw/pkg/snapshot/clonetree_pci.go | 148 ++++++++++ .../jaypipes/ghw/pkg/snapshot/pack.go | 112 ++++++++ .../jaypipes/ghw/pkg/snapshot/testdata.tar.gz | Bin 0 -> 485 bytes .../jaypipes/ghw/pkg/snapshot/trace.go | 19 ++ .../jaypipes/ghw/pkg/snapshot/unpack.go | 129 +++++++++ .../jaypipes/ghw/pkg/topology/topology.go | 128 +++++++++ .../ghw/pkg/topology/topology_linux.go | 100 +++++++ .../ghw/pkg/topology/topology_stub.go | 17 ++ .../ghw/pkg/topology/topology_windows.go | 156 +++++++++++ .../jaypipes/ghw/pkg/unitutil/unit.go | 37 +++ .../github.com/jaypipes/ghw/pkg/util/util.go | 53 ++++ vendor/modules.txt | 16 +- 60 files changed, 3787 insertions(+), 18 deletions(-) create mode 100644 pkg/compute/tasks/guest_cpuset_task.go create mode 100644 pkg/hostman/hostutils/hardware/cpu.go create mode 100644 pkg/hostman/hostutils/hardware/doc.go create mode 100644 pkg/hostman/hostutils/hardware/topology.go delete mode 100644 vendor/github.com/go-ole/go-ole/go.sum create mode 100644 vendor/github.com/jaypipes/ghw/COPYING create mode 100644 vendor/github.com/jaypipes/ghw/pkg/context/context.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/cpu/cpu.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_linux.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_stub.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_windows.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/linuxpath/path_linux.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/marshal/marshal.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/memory/memory.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/memory/memory_cache.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/memory/memory_cache_linux.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/memory/memory_linux.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/memory/memory_stub.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/memory/memory_windows.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/option/option.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/pci/address/address.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_block.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_gpu.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_net.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_pci.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/pack.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/testdata.tar.gz create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/trace.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/snapshot/unpack.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/topology/topology.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/topology/topology_linux.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/topology/topology_stub.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/topology/topology_windows.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/unitutil/unit.go create mode 100644 vendor/github.com/jaypipes/ghw/pkg/util/util.go diff --git a/cmd/climc/shell/compute/hosts.go b/cmd/climc/shell/compute/hosts.go index d7ebb6f3af..3a54baf808 100644 --- a/cmd/climc/shell/compute/hosts.go +++ b/cmd/climc/shell/compute/hosts.go @@ -17,6 +17,7 @@ package compute import ( "context" "fmt" + "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -117,6 +118,33 @@ func init() { return nil }) + type HostSysInfoOpt struct { + options.BaseIdOptions + Key string `help:"The key for extract, e.g. 'cpu_info.processors'"` + Format string `help:"Output format" choices:"yaml|json" default:"yaml"` + } + + R(&HostSysInfoOpt{}, "host-sysinfo", "Get host system info", func(s *mcclient.ClientSession, args *HostSysInfoOpt) error { + obj, err := modules.Hosts.Get(s, args.GetId(), nil) + if err != nil { + return err + } + keys := []string{"sys_info"} + if args.Key != "" { + keys = append(keys, strings.Split(args.Key, ".")...) + } + sysInfo, err := obj.Get(keys...) + if err != nil { + return errors.Wrap(err, "Get sys_info") + } + if args.Format == "yaml" { + fmt.Print(sysInfo.YAMLString()) + } else { + fmt.Print(sysInfo.PrettyString()) + } + return nil + }) + type HostUpdateOptions struct { ID string `help:"ID or Name of Host"` Name string `help:"New name of the host"` diff --git a/cmd/climc/shell/compute/servers.go b/cmd/climc/shell/compute/servers.go index 302578f18d..7e1fed8b14 100644 --- a/cmd/climc/shell/compute/servers.go +++ b/cmd/climc/shell/compute/servers.go @@ -100,6 +100,8 @@ func init() { cmd.Perform("user-metadata", &baseoptions.ResourceMetadataOptions{}) cmd.Perform("set-user-metadata", &baseoptions.ResourceMetadataOptions{}) cmd.Perform("probe-isolated-devices", &options.ServerIdOptions{}) + cmd.Perform("cpuset", &options.ServerCPUSetOptions{}) + cmd.Perform("cpuset-remove", &options.ServerIdOptions{}) cmd.Get("vnc", new(options.ServerIdOptions)) cmd.Get("desc", new(options.ServerIdOptions)) diff --git a/go.mod b/go.mod index f9396c44e3..bd9cb8b543 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/RoaringBitmap/roaring v0.4.16 // indirect github.com/Shopify/sarama v1.20.0 // indirect github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect - github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/aliyun/alibaba-cloud-sdk-go v1.61.684 github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible github.com/anacrolix/dht v0.0.0-20181129074040-b09db78595aa // indirect @@ -58,7 +57,6 @@ require ( github.com/gin-gonic/gin v1.7.0 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect github.com/go-logfmt/logfmt v0.4.0 // indirect - github.com/go-ole/go-ole v1.2.2 // indirect github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/go-yaml/yaml v2.1.0+incompatible github.com/gofrs/uuid v4.1.0+incompatible // indirect @@ -77,6 +75,7 @@ require ( github.com/hako/durafmt v0.0.0-20180520121703-7b7ae1e72ead github.com/huandu/xstrings v1.2.0 // indirect github.com/imdario/mergo v0.3.6 // indirect + github.com/jaypipes/ghw v0.8.0 github.com/jdcloud-api/jdcloud-sdk-go v1.55.0 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c diff --git a/go.sum b/go.sum index c7de4d0fca..2f37247b3b 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/Shopify/sarama v1.20.0 h1:wAMHhl1lGRlobeoV/xOKpbqD2OQsOvY4A/vIOGroIe8 github.com/Shopify/sarama v1.20.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM= @@ -255,8 +255,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-ole/go-ole v1.2.2 h1:QNWhweRd9D5Py2rRVboZ2L4SEoW/dyraWJCc8bgS8kE= -github.com/go-ole/go-ole v1.2.2/go.mod h1:pnvuG7BrDMZ8ifMurTQmxwhQM/odqm9sSqNe5BUI7v4= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= @@ -379,6 +379,9 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jaypipes/ghw v0.8.0 h1:02q1pTm9CD83vuhBsEZZhOCS128pq87uyaQeJZkp3sQ= +github.com/jaypipes/ghw v0.8.0/go.mod h1:+gR9bjm3W/HnFi90liF+Fj9GpCe/Dsibl9Im8KmC7c4= +github.com/jaypipes/pcidb v0.6.0/go.mod h1:L2RGk04sfRhp5wvHO0gfRAMoLY/F3PKv/nwJeVoho0o= github.com/jdcloud-api/jdcloud-sdk-go v1.55.0 h1:mzVj8r6fluEwjn8ogqtGfYW2qSIVUaEq0JAsvjCav3A= github.com/jdcloud-api/jdcloud-sdk-go v1.55.0/go.mod h1:UrKjuULIWLjHFlG6aSPunArE5QX57LftMmStAZJBEX8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -488,6 +491,7 @@ github.com/minio/minio-go/v6 v6.0.33 h1:CNThMAZ9bN6OEIP8DulqlYpaXKJ18rgG/Cqm14Q4 github.com/minio/minio-go/v6 v6.0.33/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= @@ -598,6 +602,7 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -923,6 +928,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= k8s.io/api v0.19.3 h1:GN6ntFnv44Vptj/b+OnMW7FmzkpDoIDLZRvKX3XH9aU= k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= k8s.io/apimachinery v0.19.3 h1:bpIQXlKjB4cB/oNpnNnV+BybGPR7iP5oYpsOTEJ4hgc= diff --git a/pkg/apis/compute/guest_const.go b/pkg/apis/compute/guest_const.go index 4002d7234d..c006f467f2 100644 --- a/pkg/apis/compute/guest_const.go +++ b/pkg/apis/compute/guest_const.go @@ -330,6 +330,7 @@ const ( VM_METADATA_OS_DISTRO = "os_distribution" VM_METADATA_OS_NAME = "os_name" VM_METADATA_OS_VERSION = "os_version" + VM_METADATA_CGROUP_CPUSET = "cgroup_cpuset" ) func Hypervisors2HostTypes(hypervisors []string) []string { diff --git a/pkg/apis/compute/guests.go b/pkg/apis/compute/guests.go index 77f40676d0..3a2a0ce5ab 100644 --- a/pkg/apis/compute/guests.go +++ b/pkg/apis/compute/guests.go @@ -797,3 +797,17 @@ type ServerCreateSnapshotParams struct { Name string `json:"name"` GenerateName string `json:"generate_name"` } + +type ServerCPUSetInput struct { + // Specifies the CPUs that tasks in this cgroup are permitted to access. + CPUS []int `json:"cpus"` +} + +type ServerCPUSetResp struct{} + +type ServerCPUSetRemoveInput struct{} + +type ServerCPUSetRemoveResp struct { + Done bool `json:"done"` + Error string `json:"error"` +} diff --git a/pkg/apis/host/types.go b/pkg/apis/host/types.go index 5b0d581ef9..6f6d711a80 100644 --- a/pkg/apis/host/types.go +++ b/pkg/apis/host/types.go @@ -14,7 +14,20 @@ package host +import ( + "github.com/jaypipes/ghw/pkg/cpu" + "github.com/jaypipes/ghw/pkg/topology" +) + type ServerCloneDiskFromStorageResponse struct { TargetAccessPath string `json:"target_access_path"` TargetFormat string `json:"target_format"` } + +type HostTopology struct { + *topology.Info +} + +type HostCPUInfo struct { + *cpu.Info +} diff --git a/pkg/cloudcommon/db/opslog_const.go b/pkg/cloudcommon/db/opslog_const.go index c086ee890e..29bcb239db 100644 --- a/pkg/cloudcommon/db/opslog_const.go +++ b/pkg/cloudcommon/db/opslog_const.go @@ -227,6 +227,11 @@ const ( ACT_GUEST_SRC_CHECK = "guest_src_check" + ACT_GUEST_CPUSET = "guest_cpuset" + ACT_GUEST_CPUSET_FAIL = "guest_cpuset_fail" + ACT_GUEST_CPUSET_REMOVE = "guest_cpuset_remove" + ACT_GUEST_CPUSET_REMOVE_FAIL = "guest_cpuset_remove_fail" + ACT_CHANGE_BANDWIDTH = "eip_change_bandwidth" ACT_EIP_CONVERT_FAIL = "eip_convert_fail" diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 62918f2257..de5aa7053a 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -453,3 +453,11 @@ func (self *SBaseGuestDriver) RequestSyncIsolatedDevice(ctx context.Context, gue task.ScheduleRun(nil) return nil } + +func (self *SBaseGuestDriver) RequestCPUSet(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, guest *models.SGuest, input *api.ServerCPUSetInput) (*api.ServerCPUSetResp, error) { + return nil, httperrors.ErrNotImplemented +} + +func (self *SBaseGuestDriver) RequestCPUSetRemove(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, guest *models.SGuest, input *api.ServerCPUSetRemoveInput) error { + return httperrors.ErrNotImplemented +} diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 06728b910a..c444bd89c5 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -925,3 +925,34 @@ func (self *SKVMGuestDriver) ValidateUpdateData(ctx context.Context, guest *mode func (self *SKVMGuestDriver) RequestSyncIsolatedDevice(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } + +func (self *SKVMGuestDriver) RequestCPUSet(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, guest *models.SGuest, input *api.ServerCPUSetInput) (*api.ServerCPUSetResp, error) { + url := fmt.Sprintf("%s/servers/%s/cpuset", host.ManagerUri, guest.Id) + httpClient := httputils.GetDefaultClient() + header := mcclient.GetTokenHeaders(userCred) + body := jsonutils.Marshal(input) + _, respBody, err := httputils.JSONRequest(httpClient, ctx, "POST", url, header, body, false) + if err != nil { + return nil, errors.Wrap(err, "host request") + } + resp := new(api.ServerCPUSetResp) + if respBody == nil { + return resp, nil + } + if err := respBody.Unmarshal(resp); err != nil { + return nil, errors.Wrap(err, "unmarshal response") + } + return resp, nil +} + +func (self *SKVMGuestDriver) RequestCPUSetRemove(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, guest *models.SGuest, input *api.ServerCPUSetRemoveInput) error { + url := fmt.Sprintf("%s/servers/%s/cpuset-remove", host.ManagerUri, guest.Id) + httpClient := httputils.GetDefaultClient() + header := mcclient.GetTokenHeaders(userCred) + body := jsonutils.Marshal(input) + _, _, err := httputils.JSONRequest(httpClient, ctx, "POST", url, header, body, false) + if err != nil { + return errors.Wrap(err, "host request") + } + return nil +} diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index f7814d2b09..4ed834fc7c 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -40,6 +40,7 @@ import ( "yunion.io/x/onecloud/pkg/apis" billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" imageapi "yunion.io/x/onecloud/pkg/apis/image" noapi "yunion.io/x/onecloud/pkg/apis/notify" schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" @@ -5346,3 +5347,69 @@ func (self *SGuest) PerformProbeIsolatedDevices(ctx context.Context, userCred mc } return jsonutils.Marshal(devs), nil } + +func (self *SGuest) PerformCpuset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *api.ServerCPUSetInput) (jsonutils.JSONObject, error) { + host, err := self.GetHost() + if err != nil { + return nil, errors.Wrap(err, "get host model") + } + topoObj, err := host.SysInfo.Get("topology") + if err != nil { + return nil, errors.Wrap(err, "get topology from host sys_info") + } + + hostTopo := new(hostapi.HostTopology) + if err := topoObj.Unmarshal(hostTopo); err != nil { + return nil, errors.Wrap(err, "Unmarshal host topology struct") + } + + // get host logical cores + allCores := []int{} + for _, node := range hostTopo.Nodes { + for _, cores := range node.Cores { + allCores = append(allCores, cores.LogicalProcessors...) + } + } + + if !sets.NewInt(allCores...).HasAll(data.CPUS...) { + return nil, httperrors.NewInputParameterError("Host cores %v not contains input %v", allCores, data.CPUS) + } + + if err := self.SetMetadata(ctx, api.VM_METADATA_CGROUP_CPUSET, data, userCred); err != nil { + return nil, errors.Wrap(err, "set metadata") + } + + return nil, self.StartGuestCPUSetTask(ctx, userCred, data) +} + +func (self *SGuest) StartGuestCPUSetTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCPUSetInput) error { + task, err := taskman.TaskManager.NewTask(ctx, "GuestCPUSetTask", self, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), "", "") + if err != nil { + return errors.Wrap(err, "New GuestCPUSetTask") + } + return task.ScheduleRun(nil) +} + +func (self *SGuest) PerformCpusetRemove(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *api.ServerCPUSetRemoveInput) (*api.ServerCPUSetRemoveResp, error) { + if err := self.RemoveMetadata(ctx, api.VM_METADATA_CGROUP_CPUSET, userCred); err != nil { + return nil, errors.Wrapf(err, "remove metadata %q", api.VM_METADATA_CGROUP_CPUSET) + } + host, err := self.GetHost() + if err != nil { + return nil, errors.Wrap(err, "get host model") + } + + // TODO: maybe change to async task + db.OpsLog.LogEvent(self, db.ACT_GUEST_CPUSET_REMOVE, nil, userCred) + resp := new(api.ServerCPUSetRemoveResp) + if err := self.GetDriver().RequestCPUSetRemove(ctx, userCred, host, self, data); err != nil { + db.OpsLog.LogEvent(self, db.ACT_GUEST_CPUSET_REMOVE_FAIL, err, userCred) + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_VM_CPUSET_REMOVE, data, userCred, false) + resp.Error = err.Error() + } else { + db.OpsLog.LogEvent(self, db.ACT_GUEST_CPUSET_REMOVE, nil, userCred) + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_VM_CPUSET_REMOVE, data, userCred, true) + resp.Done = true + } + return resp, nil +} diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index c8b95ff349..6d75c1d9df 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -217,6 +217,9 @@ type IGuestDriver interface { RequestChangeDiskStorage(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, input *api.ServerChangeDiskStorageInternalInput, task taskman.ITask) error RequestSyncIsolatedDevice(ctx context.Context, guest *SGuest, task taskman.ITask) error + + RequestCPUSet(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, guest *SGuest, input *api.ServerCPUSetInput) (*api.ServerCPUSetResp, error) + RequestCPUSetRemove(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, guest *SGuest, input *api.ServerCPUSetRemoveInput) error } var guestDrivers map[string]IGuestDriver diff --git a/pkg/compute/tasks/guest_cpuset_task.go b/pkg/compute/tasks/guest_cpuset_task.go new file mode 100644 index 0000000000..07820474bd --- /dev/null +++ b/pkg/compute/tasks/guest_cpuset_task.go @@ -0,0 +1,67 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type GuestCPUSetTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestCPUSetTask{}) +} + +func (self *GuestCPUSetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + self.SetStage("OnSyncComplete", nil) + db.OpsLog.LogEvent(guest, db.ACT_GUEST_CPUSET, self.GetParams(), self.UserCred) + if err := guest.StartSyncTask(ctx, self.GetUserCred(), true, self.GetTaskId()); err != nil { + self.setStageFailed(ctx, guest, data) + } +} + +func (self *GuestCPUSetTask) setStageFailed(ctx context.Context, obj *models.SGuest, data jsonutils.JSONObject) { + logclient.AddActionLogWithStartable(self, obj, logclient.ACT_VM_CPUSET, data, self.UserCred, false) + db.OpsLog.LogEvent(obj, db.ACT_GUEST_CPUSET_FAIL, data, self.UserCred) + self.SetStageFailed(ctx, data) +} + +func (self *GuestCPUSetTask) OnSyncComplete(ctx context.Context, obj *models.SGuest, data jsonutils.JSONObject) { + host, _ := obj.GetHost() + input := new(api.ServerCPUSetInput) + self.GetParams().Unmarshal(input) + _, err := obj.GetDriver().RequestCPUSet(ctx, self.GetUserCred(), host, obj, input) + if err != nil { + self.setStageFailed(ctx, obj, jsonutils.NewString(err.Error())) + return + } + logclient.AddActionLogWithStartable(self, obj, logclient.ACT_VM_CPUSET, input, self.UserCred, true) + self.SetStageComplete(ctx, nil) +} + +func (self *GuestCPUSetTask) OnSyncCompleteFailed(ctx context.Context, obj *models.SGuest, data jsonutils.JSONObject) { + self.setStageFailed(ctx, obj, data) +} diff --git a/pkg/hostman/guestman/guesthandlers/guesthandler.go b/pkg/hostman/guestman/guesthandlers/guesthandler.go index 1c5cfcb3f9..917fdc84d7 100644 --- a/pkg/hostman/guestman/guesthandlers/guesthandler.go +++ b/pkg/hostman/guestman/guesthandlers/guesthandler.go @@ -84,6 +84,8 @@ func AddGuestTaskHandler(prefix string, app *appsrv.Application) { "list-forward": guestListForward, "close-forward": guestCloseForward, "storage-clone-disk": guestStorageCloneDisk, + "cpuset": guestCPUSet, + "cpuset-remove": guestCPUSetRemove, } { app.AddHandler("POST", fmt.Sprintf("%s/%s//%s", prefix, keyWord, action), @@ -625,3 +627,20 @@ func guestStorageCloneDisk(ctx context.Context, sid string, body jsonutils.JSONO hostutils.DelayTaskWithoutReqctx(ctx, guestman.GetGuestManager().StorageCloneDisk, params) return nil, nil } + +func guestCPUSet(ctx context.Context, sid string, body jsonutils.JSONObject) (interface{}, error) { + input := new(computeapi.ServerCPUSetInput) + if err := body.Unmarshal(input); err != nil { + return nil, err + } + gm := guestman.GetGuestManager() + return gm.CPUSet(ctx, sid, input) +} + +func guestCPUSetRemove(ctx context.Context, sid string, body jsonutils.JSONObject) (interface{}, error) { + gm := guestman.GetGuestManager() + if err := gm.CPUSetRemove(ctx, sid); err != nil { + return nil, err + } + return nil, nil +} diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 271e4dd9a3..983e90f57f 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -254,6 +254,22 @@ func (m *SGuestManager) cpusetBalance() { } } +func (m *SGuestManager) CPUSet(ctx context.Context, sid string, req *compute.ServerCPUSetInput) (*compute.ServerCPUSetResp, error) { + guest, ok := m.GetServer(sid) + if !ok { + return nil, httperrors.NewNotFoundError("Not found") + } + return guest.CPUSet(ctx, req) +} + +func (m *SGuestManager) CPUSetRemove(ctx context.Context, sid string) error { + guest, ok := m.GetServer(sid) + if !ok { + return httperrors.NewNotFoundError("Not found") + } + return guest.CPUSetRemove(ctx) +} + func (m *SGuestManager) IsGuestDir(f os.FileInfo) bool { if !regutils.MatchUUID(f.Name()) { return false diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 76e584472c..fb383e8d0e 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -1446,6 +1446,7 @@ func (s *SKVMGuestInstance) SetCgroup() { s.cgroupPid = s.GetPid() s.setCgroupIo() s.setCgroupCpu() + s.setCgroupCPUSet() } func (s *SKVMGuestInstance) setCgroupIo() { @@ -1475,6 +1476,31 @@ func (s *SKVMGuestInstance) setCgroupCpu() { cgrouputils.CgroupSet(strconv.Itoa(s.cgroupPid), int(cpu)*cpuWeight) } +func (s *SKVMGuestInstance) setCgroupCPUSet() { + meta, _ := s.Desc.Get("metadata") + if meta == nil { + return + } + cpusetStr, _ := meta.GetString(api.VM_METADATA_CGROUP_CPUSET) + if len(cpusetStr) == 0 { + return + } + obj, err := jsonutils.ParseString(cpusetStr) + if err != nil { + log.Errorf("Parse cpusetStr %q error: %v", cpusetStr, err) + return + } + input := new(api.ServerCPUSetInput) + if err := obj.Unmarshal(input); err != nil { + log.Errorf("Unmarshal %q to ServerCPUSetInput: %v", obj, err) + return + } + if _, err := s.CPUSet(context.Background(), input); err != nil { + log.Errorf("Do CPUSet error: %v", err) + return + } +} + func (s *SKVMGuestInstance) CreateFromDesc(desc jsonutils.JSONObject) error { if err := s.PrepareDir(); err != nil { uuid, _ := desc.GetString("uuid") @@ -1839,3 +1865,38 @@ func (s *SKVMGuestInstance) getQemuCmdlineFromContent(content string) (string, e } return cmdStr, nil } + +func (s *SKVMGuestInstance) CPUSet(ctx context.Context, input *api.ServerCPUSetInput) (*api.ServerCPUSetResp, error) { + if !s.IsRunning() { + return nil, nil + } + cpus := []string{} + for _, id := range input.CPUS { + cpus = append(cpus, fmt.Sprintf("%d", id)) + } + task := cgrouputils.NewCGroupCPUSetTask(strconv.Itoa(s.GetPid()), 0, strings.Join(cpus, ",")) + if !task.SetTask() { + return nil, errors.Errorf("Cgroup cpuset task failed") + } + return new(api.ServerCPUSetResp), nil +} + +func (s *SKVMGuestInstance) CPUSetRemove(ctx context.Context) error { + metadata, err := s.Desc.Get("metadata") + if err != nil { + return errors.Wrap(err, "get metadata from desc") + } + metadata.(*jsonutils.JSONDict).Remove(api.VM_METADATA_CGROUP_CPUSET) + s.Desc.Set("metadata", metadata) + if err := s.SaveDesc(s.Desc); err != nil { + return errors.Wrap(err, "save desc after update metadata") + } + if !s.IsRunning() { + return nil + } + task := cgrouputils.NewCGroupCPUSetTask(strconv.Itoa(s.GetPid()), 0, "") + if !task.RemoveTask() { + return errors.Errorf("Remove task error happened, please lookup host log") + } + return nil +} diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 22b49ee6cd..f9a21fa822 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -44,6 +44,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts" "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/hostman/hostutils/hardware" "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/hostman/options" @@ -458,6 +459,12 @@ func (h *SHostInfo) detectHostInfo() error { h.detectStorageSystem() + topoInfo, err := hardware.GetTopology() + if err != nil { + return errors.Wrap(err, "Get hardware topology") + } + h.sysinfo.Topology = topoInfo + system_service.Init() if options.HostOptions.CheckSystemServices { if err := h.checkSystemServices(); err != nil { diff --git a/pkg/hostman/hostinfo/hostinfohelper.go b/pkg/hostman/hostinfo/hostinfohelper.go index 77072c10c6..a54ee30beb 100644 --- a/pkg/hostman/hostinfo/hostinfohelper.go +++ b/pkg/hostman/hostinfo/hostinfohelper.go @@ -32,6 +32,7 @@ import ( "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/regutils" + hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostdhcp" @@ -324,6 +325,8 @@ type SSysInfo struct { HugepagesOption string `json:"hugepages_option"` HugepageSizeKb int `json:"hugepage_size_kb"` + + Topology *hostapi.HostTopology `json:"topology"` } func StartDetachStorages(hs []jsonutils.JSONObject) { diff --git a/pkg/hostman/hostutils/hardware/cpu.go b/pkg/hostman/hostutils/hardware/cpu.go new file mode 100644 index 0000000000..dfd45abbf6 --- /dev/null +++ b/pkg/hostman/hostutils/hardware/cpu.go @@ -0,0 +1,31 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "github.com/jaypipes/ghw/pkg/cpu" + + "yunion.io/x/onecloud/pkg/apis/host" +) + +func GetCPU() (*host.HostCPUInfo, error) { + info, err := cpu.New() + if err != nil { + return nil, err + } + return &host.HostCPUInfo{ + Info: info, + }, nil +} diff --git a/pkg/hostman/hostutils/hardware/doc.go b/pkg/hostman/hostutils/hardware/doc.go new file mode 100644 index 0000000000..63cc83b099 --- /dev/null +++ b/pkg/hostman/hostutils/hardware/doc.go @@ -0,0 +1 @@ +package hardware // import "yunion.io/x/onecloud/pkg/hostman/hostutils/hardware" diff --git a/pkg/hostman/hostutils/hardware/topology.go b/pkg/hostman/hostutils/hardware/topology.go new file mode 100644 index 0000000000..06d60d167a --- /dev/null +++ b/pkg/hostman/hostutils/hardware/topology.go @@ -0,0 +1,31 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "github.com/jaypipes/ghw/pkg/topology" + + "yunion.io/x/onecloud/pkg/apis/host" +) + +func GetTopology() (*host.HostTopology, error) { + info, err := topology.New() + if err != nil { + return nil, err + } + return &host.HostTopology{ + Info: info, + }, nil +} diff --git a/pkg/mcclient/options/compute/servers.go b/pkg/mcclient/options/compute/servers.go index 4aec414fb5..a5d6580d2f 100644 --- a/pkg/mcclient/options/compute/servers.go +++ b/pkg/mcclient/options/compute/servers.go @@ -29,6 +29,7 @@ import ( schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/mcclient/options" + "yunion.io/x/onecloud/pkg/util/cgrouputils" ) var ErrEmtptyUpdate = errors.New("No valid update data") @@ -1176,3 +1177,27 @@ type ServerChangeDiskStorageOptions struct { func (o *ServerChangeDiskStorageOptions) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(o), nil } + +type ServerCPUSetOptions struct { + options.BaseIdOptions + SETS string `help:"Cgroup cpusets CPUs spec string, e.g. '0-2,16'"` +} + +func (o *ServerCPUSetOptions) Params() (jsonutils.JSONObject, error) { + sets := cgrouputils.ParseCpusetStr(o.SETS) + parts := strings.Split(sets, ",") + if len(parts) == 0 { + return nil, errors.New(fmt.Sprintf("Invalid cpu sets %q", o.SETS)) + } + input := &computeapi.ServerCPUSetInput{ + CPUS: make([]int, 0), + } + for _, s := range parts { + sd, err := strconv.Atoi(s) + if err != nil { + return nil, errors.New(fmt.Sprintf("Not digit part %q", s)) + } + input.CPUS = append(input.CPUS, sd) + } + return jsonutils.Marshal(input), nil +} diff --git a/pkg/util/logclient/consts.go b/pkg/util/logclient/consts.go index 3051a68372..01aa6751c4 100644 --- a/pkg/util/logclient/consts.go +++ b/pkg/util/logclient/consts.go @@ -67,6 +67,8 @@ const ( ACT_VM_ASSIGNSECGROUP = "vm_assignsecgroup" ACT_VM_REVOKESECGROUP = "vm_revokesecgroup" ACT_VM_SETSECGROUP = "vm_setsecgroup" + ACT_VM_CPUSET = "vm_cpuset" + ACT_VM_CPUSET_REMOVE = "vm_cpuset_remove" ACT_RESET_DISK = "reset_disk" ACT_SYNC_STATUS = "sync_status" ACT_SYNC_CONF = "sync_conf" diff --git a/vendor/github.com/StackExchange/wmi/swbemservices.go b/vendor/github.com/StackExchange/wmi/swbemservices.go index 9765a53f74..3ff8756303 100644 --- a/vendor/github.com/StackExchange/wmi/swbemservices.go +++ b/vendor/github.com/StackExchange/wmi/swbemservices.go @@ -77,7 +77,7 @@ func (s *SWbemServices) process(initError chan error) { //fmt.Println("process: starting background thread initialization") //All OLE/WMI calls must happen on the same initialized thead, so lock this goroutine runtime.LockOSThread() - defer runtime.LockOSThread() + defer runtime.UnlockOSThread() err := ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED) if err != nil { diff --git a/vendor/github.com/StackExchange/wmi/wmi.go b/vendor/github.com/StackExchange/wmi/wmi.go index a951b1258b..eab18cbfee 100644 --- a/vendor/github.com/StackExchange/wmi/wmi.go +++ b/vendor/github.com/StackExchange/wmi/wmi.go @@ -285,6 +285,10 @@ func (c *Client) loadEntity(dst interface{}, src *ole.IDispatch) (errFieldMismat } defer prop.Clear() + if prop.VT == 0x1 { //VT_NULL + continue + } + switch val := prop.Value().(type) { case int8, int16, int32, int64, int: v := reflect.ValueOf(val).Int() @@ -383,7 +387,7 @@ func (c *Client) loadEntity(dst interface{}, src *ole.IDispatch) (errFieldMismat } f.Set(fArr) } - case reflect.Uint8: + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: safeArray := prop.ToArray() if safeArray != nil { arr := safeArray.ToValueArray() @@ -394,6 +398,17 @@ func (c *Client) loadEntity(dst interface{}, src *ole.IDispatch) (errFieldMismat } f.Set(fArr) } + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + safeArray := prop.ToArray() + if safeArray != nil { + arr := safeArray.ToValueArray() + fArr := reflect.MakeSlice(f.Type(), len(arr), len(arr)) + for i, v := range arr { + s := fArr.Index(i) + s.SetInt(reflect.ValueOf(v).Int()) + } + f.Set(fArr) + } default: return &ErrFieldMismatch{ StructType: of.Type(), diff --git a/vendor/github.com/go-ole/go-ole/.travis.yml b/vendor/github.com/go-ole/go-ole/.travis.yml index 0c2c02bdf2..28f740cd5d 100644 --- a/vendor/github.com/go-ole/go-ole/.travis.yml +++ b/vendor/github.com/go-ole/go-ole/.travis.yml @@ -2,8 +2,7 @@ language: go sudo: false go: - - 1.1 - - 1.2 - - 1.3 - - 1.4 + - 1.9.x + - 1.10.x + - 1.11.x - tip diff --git a/vendor/github.com/go-ole/go-ole/go.mod b/vendor/github.com/go-ole/go-ole/go.mod index 645d3b7562..df98533ea9 100644 --- a/vendor/github.com/go-ole/go-ole/go.mod +++ b/vendor/github.com/go-ole/go-ole/go.mod @@ -1,5 +1,3 @@ module github.com/go-ole/go-ole go 1.12 - -require github.com/go-ole/go-ole v1.2.2 diff --git a/vendor/github.com/go-ole/go-ole/go.sum b/vendor/github.com/go-ole/go-ole/go.sum deleted file mode 100644 index 6a8d3b0448..0000000000 --- a/vendor/github.com/go-ole/go-ole/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/go-ole/go-ole v1.2.2 h1:HXmymm3IQ8iAfpqlbbUGLHd+SZrnmI4y1pv+WL/3R7c= -github.com/go-ole/go-ole v1.2.2/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= diff --git a/vendor/github.com/jaypipes/ghw/COPYING b/vendor/github.com/jaypipes/ghw/COPYING new file mode 100644 index 0000000000..68c771a099 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/COPYING @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/vendor/github.com/jaypipes/ghw/pkg/context/context.go b/vendor/github.com/jaypipes/ghw/pkg/context/context.go new file mode 100644 index 0000000000..315c2d8bb0 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/context/context.go @@ -0,0 +1,128 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package context + +import ( + "github.com/jaypipes/ghw/pkg/option" + "github.com/jaypipes/ghw/pkg/snapshot" +) + +// Concrete merged set of configuration switches that act as an execution +// context when calling internal discovery methods +type Context struct { + Chroot string + EnableTools bool + SnapshotPath string + SnapshotRoot string + SnapshotExclusive bool + snapshotUnpackedPath string + alert option.Alerter +} + +// New returns a Context struct pointer that has had various options set on it +func New(opts ...*option.Option) *Context { + merged := option.Merge(opts...) + ctx := &Context{ + alert: option.EnvOrDefaultAlerter(), + Chroot: *merged.Chroot, + } + + if merged.Snapshot != nil { + ctx.SnapshotPath = merged.Snapshot.Path + // root is optional, so a extra check is warranted + if merged.Snapshot.Root != nil { + ctx.SnapshotRoot = *merged.Snapshot.Root + } + ctx.SnapshotExclusive = merged.Snapshot.Exclusive + } + + if merged.Alerter != nil { + ctx.alert = merged.Alerter + } + + if merged.EnableTools != nil { + ctx.EnableTools = *merged.EnableTools + } + + return ctx +} + +// FromEnv returns an Option that has been populated from the environs or +// default options values +func FromEnv() *Context { + chrootVal := option.EnvOrDefaultChroot() + enableTools := option.EnvOrDefaultTools() + snapPathVal := option.EnvOrDefaultSnapshotPath() + snapRootVal := option.EnvOrDefaultSnapshotRoot() + snapExclusiveVal := option.EnvOrDefaultSnapshotExclusive() + return &Context{ + Chroot: chrootVal, + EnableTools: enableTools, + SnapshotPath: snapPathVal, + SnapshotRoot: snapRootVal, + SnapshotExclusive: snapExclusiveVal, + } +} + +// Do wraps a Setup/Teardown pair around the given function +func (ctx *Context) Do(fn func() error) error { + err := ctx.Setup() + if err != nil { + return err + } + defer ctx.Teardown() + return fn() +} + +// Setup prepares the extra optional data a Context may use. +// `Context`s are ready to use once returned by `New`. Optional features, +// like snapshot unpacking, may require extra steps. Run `Setup` to perform them. +// You should call `Setup` just once. It is safe to call `Setup` if you don't make +// use of optional extra features - `Setup` will do nothing. +func (ctx *Context) Setup() error { + if ctx.SnapshotPath == "" { + // nothing to do! + return nil + } + + var err error + root := ctx.SnapshotRoot + if root == "" { + root, err = snapshot.Unpack(ctx.SnapshotPath) + if err == nil { + ctx.snapshotUnpackedPath = root + } + } else { + var flags uint + if ctx.SnapshotExclusive { + flags |= snapshot.OwnTargetDirectory + } + _, err = snapshot.UnpackInto(ctx.SnapshotPath, root, flags) + } + if err != nil { + return err + } + + ctx.Chroot = root + return nil +} + +// Teardown releases any resource acquired by Setup. +// You should always call `Teardown` if you called `Setup` to free any resources +// acquired by `Setup`. Check `Do` for more automated management. +func (ctx *Context) Teardown() error { + if ctx.snapshotUnpackedPath == "" { + // if the client code provided the unpack directory, + // then it is also in charge of the cleanup. + return nil + } + return snapshot.Cleanup(ctx.snapshotUnpackedPath) +} + +func (ctx *Context) Warn(msg string, args ...interface{}) { + ctx.alert.Printf("WARNING: "+msg, args...) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu.go b/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu.go new file mode 100644 index 0000000000..2fa0cd2d06 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu.go @@ -0,0 +1,169 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package cpu + +import ( + "fmt" + + "github.com/jaypipes/ghw/pkg/context" + "github.com/jaypipes/ghw/pkg/marshal" + "github.com/jaypipes/ghw/pkg/option" +) + +// ProcessorCore describes a physical host processor core. A processor core is +// a separate processing unit within some types of central processing units +// (CPU). +type ProcessorCore struct { + // ID is the `uint32` identifier that the host gave this core. Note that + // this does *not* necessarily equate to a zero-based index of the core + // within a physical package. For example, the core IDs for an Intel Core + // i7 are 0, 1, 2, 8, 9, and 10 + ID int `json:"id"` + // Index is the zero-based index of the core on the physical processor + // package + Index int `json:"index"` + // NumThreads is the number of hardware threads associated with the core + NumThreads uint32 `json:"total_threads"` + // LogicalProcessors is a slice of ints representing the logical processor + // IDs assigned to any processing unit for the core + LogicalProcessors []int `json:"logical_processors"` +} + +// String returns a short string indicating important information about the +// processor core +func (c *ProcessorCore) String() string { + return fmt.Sprintf( + "processor core #%d (%d threads), logical processors %v", + c.Index, + c.NumThreads, + c.LogicalProcessors, + ) +} + +// Processor describes a physical host central processing unit (CPU). +type Processor struct { + // ID is the physical processor `uint32` ID according to the system + ID int `json:"id"` + // NumCores is the number of physical cores in the processor package + NumCores uint32 `json:"total_cores"` + // NumThreads is the number of hardware threads in the processor package + NumThreads uint32 `json:"total_threads"` + // Vendor is a string containing the vendor name + Vendor string `json:"vendor"` + // Model` is a string containing the vendor's model name + Model string `json:"model"` + // Capabilities is a slice of strings indicating the features the processor + // has enabled + Capabilities []string `json:"capabilities"` + // Cores is a slice of ProcessorCore` struct pointers that are packed onto + // this physical processor + Cores []*ProcessorCore `json:"cores"` +} + +// HasCapability returns true if the Processor has the supplied cpuid +// capability, false otherwise. Example of cpuid capabilities would be 'vmx' or +// 'sse4_2'. To see a list of potential cpuid capabilitiies, see the section on +// CPUID feature bits in the following article: +// +// https://en.wikipedia.org/wiki/CPUID +func (p *Processor) HasCapability(find string) bool { + for _, c := range p.Capabilities { + if c == find { + return true + } + } + return false +} + +// String returns a short string describing the Processor +func (p *Processor) String() string { + ncs := "cores" + if p.NumCores == 1 { + ncs = "core" + } + nts := "threads" + if p.NumThreads == 1 { + nts = "thread" + } + return fmt.Sprintf( + "physical package #%d (%d %s, %d hardware %s)", + p.ID, + p.NumCores, + ncs, + p.NumThreads, + nts, + ) +} + +// Info describes all central processing unit (CPU) functionality on a host. +// Returned by the `ghw.CPU()` function. +type Info struct { + ctx *context.Context + // TotalCores is the total number of physical cores the host system + // contains + TotalCores uint32 `json:"total_cores"` + // TotalThreads is the total number of hardware threads the host system + // contains + TotalThreads uint32 `json:"total_threads"` + // Processors is a slice of Processor struct pointers, one for each + // physical processor package contained in the host + Processors []*Processor `json:"processors"` +} + +// New returns a pointer to an Info struct that contains information about the +// CPUs on the host system +func New(opts ...*option.Option) (*Info, error) { + ctx := context.New(opts...) + info := &Info{ctx: ctx} + if err := ctx.Do(info.load); err != nil { + return nil, err + } + return info, nil +} + +// String returns a short string indicating a summary of CPU information +func (i *Info) String() string { + nps := "packages" + if len(i.Processors) == 1 { + nps = "package" + } + ncs := "cores" + if i.TotalCores == 1 { + ncs = "core" + } + nts := "threads" + if i.TotalThreads == 1 { + nts = "thread" + } + return fmt.Sprintf( + "cpu (%d physical %s, %d %s, %d hardware %s)", + len(i.Processors), + nps, + i.TotalCores, + ncs, + i.TotalThreads, + nts, + ) +} + +// simple private struct used to encapsulate cpu information in a top-level +// "cpu" YAML/JSON map/object key +type cpuPrinter struct { + Info *Info `json:"cpu"` +} + +// YAMLString returns a string with the cpu information formatted as YAML +// under a top-level "cpu:" key +func (i *Info) YAMLString() string { + return marshal.SafeYAML(i.ctx, cpuPrinter{i}) +} + +// JSONString returns a string with the cpu information formatted as JSON +// under a top-level "cpu:" key +func (i *Info) JSONString(indent bool) string { + return marshal.SafeJSON(i.ctx, cpuPrinter{i}, indent) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_linux.go b/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_linux.go new file mode 100644 index 0000000000..44e4ced745 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_linux.go @@ -0,0 +1,220 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package cpu + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/context" + "github.com/jaypipes/ghw/pkg/linuxpath" + "github.com/jaypipes/ghw/pkg/util" +) + +func (i *Info) load() error { + i.Processors = processorsGet(i.ctx) + var totCores uint32 + var totThreads uint32 + for _, p := range i.Processors { + totCores += p.NumCores + totThreads += p.NumThreads + } + i.TotalCores = totCores + i.TotalThreads = totThreads + return nil +} + +func processorsGet(ctx *context.Context) []*Processor { + procs := make([]*Processor, 0) + paths := linuxpath.New(ctx) + + r, err := os.Open(paths.ProcCpuinfo) + if err != nil { + return nil + } + defer util.SafeClose(r) + + // An array of maps of attributes describing the logical processor + procAttrs := make([]map[string]string, 0) + curProcAttrs := make(map[string]string) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + // Output of /proc/cpuinfo has a blank newline to separate logical + // processors, so here we collect up all the attributes we've + // collected for this logical processor block + procAttrs = append(procAttrs, curProcAttrs) + // Reset the current set of processor attributes... + curProcAttrs = make(map[string]string) + continue + } + parts := strings.Split(line, ":") + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + curProcAttrs[key] = value + } + + // Build a set of physical processor IDs which represent the physical + // package of the CPU + setPhysicalIDs := make(map[int]bool) + for _, attrs := range procAttrs { + pid, err := strconv.Atoi(attrs["physical id"]) + if err != nil { + continue + } + setPhysicalIDs[pid] = true + } + + for pid := range setPhysicalIDs { + p := &Processor{ + ID: pid, + } + // The indexes into the array of attribute maps for each logical + // processor within the physical processor + lps := make([]int, 0) + for x := range procAttrs { + lppid, err := strconv.Atoi(procAttrs[x]["physical id"]) + if err != nil { + continue + } + if pid == lppid { + lps = append(lps, x) + } + } + first := procAttrs[lps[0]] + p.Model = first["model name"] + p.Vendor = first["vendor_id"] + numCores, err := strconv.Atoi(first["cpu cores"]) + if err != nil { + continue + } + p.NumCores = uint32(numCores) + numThreads, err := strconv.Atoi(first["siblings"]) + if err != nil { + continue + } + p.NumThreads = uint32(numThreads) + + // The flags field is a space-separated list of CPU capabilities + p.Capabilities = strings.Split(first["flags"], " ") + + cores := make([]*ProcessorCore, 0) + for _, lpidx := range lps { + lpid, err := strconv.Atoi(procAttrs[lpidx]["processor"]) + if err != nil { + continue + } + coreID, err := strconv.Atoi(procAttrs[lpidx]["core id"]) + if err != nil { + continue + } + var core *ProcessorCore + for _, c := range cores { + if c.ID == coreID { + c.LogicalProcessors = append( + c.LogicalProcessors, + lpid, + ) + c.NumThreads = uint32(len(c.LogicalProcessors)) + core = c + } + } + if core == nil { + coreLps := make([]int, 1) + coreLps[0] = lpid + core = &ProcessorCore{ + ID: coreID, + Index: len(cores), + NumThreads: 1, + LogicalProcessors: coreLps, + } + cores = append(cores, core) + } + } + p.Cores = cores + procs = append(procs, p) + } + return procs +} + +func CoresForNode(ctx *context.Context, nodeID int) ([]*ProcessorCore, error) { + // The /sys/devices/system/node/nodeX directory contains a subdirectory + // called 'cpuX' for each logical processor assigned to the node. Each of + // those subdirectories contains a topology subdirectory which has a + // core_id file that indicates the 0-based identifier of the physical core + // the logical processor (hardware thread) is on. + paths := linuxpath.New(ctx) + path := filepath.Join( + paths.SysDevicesSystemNode, + fmt.Sprintf("node%d", nodeID), + ) + cores := make([]*ProcessorCore, 0) + + findCoreByID := func(coreID int) *ProcessorCore { + for _, c := range cores { + if c.ID == coreID { + return c + } + } + + c := &ProcessorCore{ + ID: coreID, + Index: len(cores), + LogicalProcessors: make([]int, 0), + } + cores = append(cores, c) + return c + } + + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, err + } + for _, file := range files { + filename := file.Name() + if !strings.HasPrefix(filename, "cpu") { + continue + } + if filename == "cpumap" || filename == "cpulist" { + // There are two files in the node directory that start with 'cpu' + // but are not subdirectories ('cpulist' and 'cpumap'). Ignore + // these files. + continue + } + // Grab the logical processor ID by cutting the integer from the + // /sys/devices/system/node/nodeX/cpuX filename + cpuPath := filepath.Join(path, filename) + procID, err := strconv.Atoi(filename[3:]) + if err != nil { + _, _ = fmt.Fprintf( + os.Stderr, + "failed to determine procID from %s. Expected integer after 3rd char.", + filename, + ) + continue + } + coreIDPath := filepath.Join(cpuPath, "topology", "core_id") + coreID := util.SafeIntFromFile(ctx, coreIDPath) + core := findCoreByID(coreID) + core.LogicalProcessors = append( + core.LogicalProcessors, + procID, + ) + } + + for _, c := range cores { + c.NumThreads = uint32(len(c.LogicalProcessors)) + } + + return cores, nil +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_stub.go b/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_stub.go new file mode 100644 index 0000000000..9ff41cd161 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_stub.go @@ -0,0 +1,17 @@ +// +build !linux,!windows +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package cpu + +import ( + "runtime" + + "github.com/pkg/errors" +) + +func (i *Info) load() error { + return errors.New("cpu.Info.load not implemented on " + runtime.GOOS) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_windows.go b/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_windows.go new file mode 100644 index 0000000000..07a7ddddbe --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/cpu/cpu_windows.go @@ -0,0 +1,55 @@ +// +build !linux +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package cpu + +import ( + "github.com/StackExchange/wmi" +) + +const wmqlProcessor = "SELECT Manufacturer, Name, NumberOfLogicalProcessors, NumberOfCores FROM Win32_Processor" + +type win32Processor struct { + Manufacturer *string + Name *string + NumberOfLogicalProcessors uint32 + NumberOfCores uint32 +} + +func (i *Info) load() error { + // Getting info from WMI + var win32descriptions []win32Processor + if err := wmi.Query(wmqlProcessor, &win32descriptions); err != nil { + return err + } + // Converting into standard structures + i.Processors = processorsGet(win32descriptions) + var totCores uint32 + var totThreads uint32 + for _, p := range i.Processors { + totCores += p.NumCores + totThreads += p.NumThreads + } + i.TotalCores = totCores + i.TotalThreads = totThreads + return nil +} + +func processorsGet(win32descriptions []win32Processor) []*Processor { + var procs []*Processor + // Converting into standard structures + for index, description := range win32descriptions { + p := &Processor{ + ID: index, + Model: *description.Name, + Vendor: *description.Manufacturer, + NumCores: description.NumberOfCores, + NumThreads: description.NumberOfLogicalProcessors, + } + procs = append(procs, p) + } + return procs +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/linuxpath/path_linux.go b/vendor/github.com/jaypipes/ghw/pkg/linuxpath/path_linux.go new file mode 100644 index 0000000000..0322f2e381 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/linuxpath/path_linux.go @@ -0,0 +1,71 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package linuxpath + +import ( + "fmt" + "path/filepath" + + "github.com/jaypipes/ghw/pkg/context" +) + +type Paths struct { + VarLog string + ProcMeminfo string + ProcCpuinfo string + SysKernelMMHugepages string + EtcMtab string + SysBlock string + SysDevicesSystemNode string + SysDevicesSystemMemory string + SysBusPciDevices string + SysClassDRM string + SysClassDMI string + SysClassNet string + RunUdevData string +} + +// New returns a new Paths struct containing filepath fields relative to the +// supplied Context +func New(ctx *context.Context) *Paths { + return &Paths{ + VarLog: filepath.Join(ctx.Chroot, "var", "log"), + ProcMeminfo: filepath.Join(ctx.Chroot, "proc", "meminfo"), + ProcCpuinfo: filepath.Join(ctx.Chroot, "proc", "cpuinfo"), + SysKernelMMHugepages: filepath.Join(ctx.Chroot, "sys", "kernel", "mm", "hugepages"), + EtcMtab: filepath.Join(ctx.Chroot, "etc", "mtab"), + SysBlock: filepath.Join(ctx.Chroot, "sys", "block"), + SysDevicesSystemNode: filepath.Join(ctx.Chroot, "sys", "devices", "system", "node"), + SysDevicesSystemMemory: filepath.Join(ctx.Chroot, "sys", "devices", "system", "memory"), + SysBusPciDevices: filepath.Join(ctx.Chroot, "sys", "bus", "pci", "devices"), + SysClassDRM: filepath.Join(ctx.Chroot, "sys", "class", "drm"), + SysClassDMI: filepath.Join(ctx.Chroot, "sys", "class", "dmi"), + SysClassNet: filepath.Join(ctx.Chroot, "sys", "class", "net"), + RunUdevData: filepath.Join(ctx.Chroot, "run", "udev", "data"), + } +} + +func (p *Paths) NodeCPU(nodeID int, lpID int) string { + return filepath.Join( + p.SysDevicesSystemNode, + fmt.Sprintf("node%d", nodeID), + fmt.Sprintf("cpu%d", lpID), + ) +} + +func (p *Paths) NodeCPUCache(nodeID int, lpID int) string { + return filepath.Join( + p.NodeCPU(nodeID, lpID), + "cache", + ) +} + +func (p *Paths) NodeCPUCacheIndex(nodeID int, lpID int, cacheIndex int) string { + return filepath.Join( + p.NodeCPUCache(nodeID, lpID), + fmt.Sprintf("index%d", cacheIndex), + ) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/marshal/marshal.go b/vendor/github.com/jaypipes/ghw/pkg/marshal/marshal.go new file mode 100644 index 0000000000..e8f1bbeac9 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/marshal/marshal.go @@ -0,0 +1,47 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package marshal + +import ( + "encoding/json" + + "github.com/ghodss/yaml" + "github.com/jaypipes/ghw/pkg/context" +) + +// safeYAML returns a string after marshalling the supplied parameter into YAML +func SafeYAML(ctx *context.Context, p interface{}) string { + b, err := json.Marshal(p) + if err != nil { + ctx.Warn("error marshalling JSON: %s", err) + return "" + } + yb, err := yaml.JSONToYAML(b) + if err != nil { + ctx.Warn("error converting JSON to YAML: %s", err) + return "" + } + return string(yb) +} + +// safeJSON returns a string after marshalling the supplied parameter into +// JSON. Accepts an optional argument to trigger pretty/indented formatting of +// the JSON string +func SafeJSON(ctx *context.Context, p interface{}, indent bool) string { + var b []byte + var err error + if !indent { + b, err = json.Marshal(p) + } else { + b, err = json.MarshalIndent(&p, "", " ") + } + if err != nil { + ctx.Warn("error marshalling JSON: %s", err) + return "" + } + return string(b) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/memory/memory.go b/vendor/github.com/jaypipes/ghw/pkg/memory/memory.go new file mode 100644 index 0000000000..93605d2e5c --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/memory/memory.go @@ -0,0 +1,80 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package memory + +import ( + "fmt" + "math" + + "github.com/jaypipes/ghw/pkg/context" + "github.com/jaypipes/ghw/pkg/marshal" + "github.com/jaypipes/ghw/pkg/option" + "github.com/jaypipes/ghw/pkg/unitutil" + "github.com/jaypipes/ghw/pkg/util" +) + +type Module struct { + Label string `json:"label"` + Location string `json:"location"` + SerialNumber string `json:"serial_number"` + SizeBytes int64 `json:"size_bytes"` + Vendor string `json:"vendor"` +} + +type Info struct { + ctx *context.Context + TotalPhysicalBytes int64 `json:"total_physical_bytes"` + TotalUsableBytes int64 `json:"total_usable_bytes"` + // An array of sizes, in bytes, of memory pages supported by the host + SupportedPageSizes []uint64 `json:"supported_page_sizes"` + Modules []*Module `json:"modules"` +} + +func New(opts ...*option.Option) (*Info, error) { + ctx := context.New(opts...) + info := &Info{ctx: ctx} + if err := ctx.Do(info.load); err != nil { + return nil, err + } + return info, nil +} + +func (i *Info) String() string { + tpbs := util.UNKNOWN + if i.TotalPhysicalBytes > 0 { + tpb := i.TotalPhysicalBytes + unit, unitStr := unitutil.AmountString(tpb) + tpb = int64(math.Ceil(float64(i.TotalPhysicalBytes) / float64(unit))) + tpbs = fmt.Sprintf("%d%s", tpb, unitStr) + } + tubs := util.UNKNOWN + if i.TotalUsableBytes > 0 { + tub := i.TotalUsableBytes + unit, unitStr := unitutil.AmountString(tub) + tub = int64(math.Ceil(float64(i.TotalUsableBytes) / float64(unit))) + tubs = fmt.Sprintf("%d%s", tub, unitStr) + } + return fmt.Sprintf("memory (%s physical, %s usable)", tpbs, tubs) +} + +// simple private struct used to encapsulate memory information in a top-level +// "memory" YAML/JSON map/object key +type memoryPrinter struct { + Info *Info `json:"memory"` +} + +// YAMLString returns a string with the memory information formatted as YAML +// under a top-level "memory:" key +func (i *Info) YAMLString() string { + return marshal.SafeYAML(i.ctx, memoryPrinter{i}) +} + +// JSONString returns a string with the memory information formatted as JSON +// under a top-level "memory:" key +func (i *Info) JSONString(indent bool) string { + return marshal.SafeJSON(i.ctx, memoryPrinter{i}, indent) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/memory/memory_cache.go b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_cache.go new file mode 100644 index 0000000000..5adbb9cd15 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_cache.go @@ -0,0 +1,101 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package memory + +import ( + "fmt" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/unitutil" +) + +type CacheType int + +const ( + CACHE_TYPE_UNIFIED CacheType = iota + CACHE_TYPE_INSTRUCTION + CACHE_TYPE_DATA +) + +var ( + memoryCacheTypeString = map[CacheType]string{ + CACHE_TYPE_UNIFIED: "Unified", + CACHE_TYPE_INSTRUCTION: "Instruction", + CACHE_TYPE_DATA: "Data", + } +) + +func (a CacheType) String() string { + return memoryCacheTypeString[a] +} + +// NOTE(jaypipes): since serialized output is as "official" as we're going to +// get, let's lowercase the string output when serializing, in order to +// "normalize" the expected serialized output +func (a CacheType) MarshalJSON() ([]byte, error) { + return []byte("\"" + strings.ToLower(a.String()) + "\""), nil +} + +type SortByCacheLevelTypeFirstProcessor []*Cache + +func (a SortByCacheLevelTypeFirstProcessor) Len() int { return len(a) } +func (a SortByCacheLevelTypeFirstProcessor) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a SortByCacheLevelTypeFirstProcessor) Less(i, j int) bool { + if a[i].Level < a[j].Level { + return true + } else if a[i].Level == a[j].Level { + if a[i].Type < a[j].Type { + return true + } else if a[i].Type == a[j].Type { + // NOTE(jaypipes): len(LogicalProcessors) is always >0 and is always + // sorted lowest LP ID to highest LP ID + return a[i].LogicalProcessors[0] < a[j].LogicalProcessors[0] + } + } + return false +} + +type SortByLogicalProcessorId []uint32 + +func (a SortByLogicalProcessorId) Len() int { return len(a) } +func (a SortByLogicalProcessorId) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a SortByLogicalProcessorId) Less(i, j int) bool { return a[i] < a[j] } + +type Cache struct { + Level uint8 `json:"level"` + Type CacheType `json:"type"` + SizeBytes uint64 `json:"size_bytes"` + // The set of logical processors (hardware threads) that have access to the + // cache + LogicalProcessors []uint32 `json:"logical_processors"` +} + +func (c *Cache) String() string { + sizeKb := c.SizeBytes / uint64(unitutil.KB) + typeStr := "" + if c.Type == CACHE_TYPE_INSTRUCTION { + typeStr = "i" + } else if c.Type == CACHE_TYPE_DATA { + typeStr = "d" + } + cacheIDStr := fmt.Sprintf("L%d%s", c.Level, typeStr) + processorMapStr := "" + if c.LogicalProcessors != nil { + lpStrings := make([]string, len(c.LogicalProcessors)) + for x, lpid := range c.LogicalProcessors { + lpStrings[x] = strconv.Itoa(int(lpid)) + } + processorMapStr = " shared with logical processors: " + strings.Join(lpStrings, ",") + } + return fmt.Sprintf( + "%s cache (%d KB)%s", + cacheIDStr, + sizeKb, + processorMapStr, + ) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/memory/memory_cache_linux.go b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_cache_linux.go new file mode 100644 index 0000000000..88ab5e56ce --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_cache_linux.go @@ -0,0 +1,188 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package memory + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/context" + "github.com/jaypipes/ghw/pkg/linuxpath" + "github.com/jaypipes/ghw/pkg/unitutil" +) + +func CachesForNode(ctx *context.Context, nodeID int) ([]*Cache, error) { + // The /sys/devices/node/nodeX directory contains a subdirectory called + // 'cpuX' for each logical processor assigned to the node. Each of those + // subdirectories containers a 'cache' subdirectory which contains a number + // of subdirectories beginning with 'index' and ending in the cache's + // internal 0-based identifier. Those subdirectories contain a number of + // files, including 'shared_cpu_list', 'size', and 'type' which we use to + // determine cache characteristics. + paths := linuxpath.New(ctx) + path := filepath.Join( + paths.SysDevicesSystemNode, + fmt.Sprintf("node%d", nodeID), + ) + caches := make(map[string]*Cache) + + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, err + } + for _, file := range files { + filename := file.Name() + if !strings.HasPrefix(filename, "cpu") { + continue + } + if filename == "cpumap" || filename == "cpulist" { + // There are two files in the node directory that start with 'cpu' + // but are not subdirectories ('cpulist' and 'cpumap'). Ignore + // these files. + continue + } + // Grab the logical processor ID by cutting the integer from the + // /sys/devices/system/node/nodeX/cpuX filename + cpuPath := filepath.Join(path, filename) + lpID, _ := strconv.Atoi(filename[3:]) + + // Inspect the caches for each logical processor. There will be a + // /sys/devices/system/node/nodeX/cpuX/cache directory containing a + // number of directories beginning with the prefix "index" followed by + // a number. The number indicates the level of the cache, which + // indicates the "distance" from the processor. Each of these + // directories contains information about the size of that level of + // cache and the processors mapped to it. + cachePath := filepath.Join(cpuPath, "cache") + if _, err = os.Stat(cachePath); errors.Is(err, os.ErrNotExist) { + continue + } + cacheDirFiles, err := ioutil.ReadDir(cachePath) + if err != nil { + return nil, err + } + for _, cacheDirFile := range cacheDirFiles { + cacheDirFileName := cacheDirFile.Name() + if !strings.HasPrefix(cacheDirFileName, "index") { + continue + } + cacheIndex, _ := strconv.Atoi(cacheDirFileName[5:]) + + // The cache information is repeated for each node, so here, we + // just ensure that we only have a one Cache object for each + // unique combination of level, type and processor map + level := memoryCacheLevel(paths, nodeID, lpID, cacheIndex) + cacheType := memoryCacheType(paths, nodeID, lpID, cacheIndex) + sharedCpuMap := memoryCacheSharedCPUMap(paths, nodeID, lpID, cacheIndex) + cacheKey := fmt.Sprintf("%d-%d-%s", level, cacheType, sharedCpuMap) + + cache, exists := caches[cacheKey] + if !exists { + size := memoryCacheSize(paths, nodeID, lpID, level) + cache = &Cache{ + Level: uint8(level), + Type: cacheType, + SizeBytes: uint64(size) * uint64(unitutil.KB), + LogicalProcessors: make([]uint32, 0), + } + caches[cacheKey] = cache + } + cache.LogicalProcessors = append( + cache.LogicalProcessors, + uint32(lpID), + ) + } + } + + cacheVals := make([]*Cache, len(caches)) + x := 0 + for _, c := range caches { + // ensure the cache's processor set is sorted by logical process ID + sort.Sort(SortByLogicalProcessorId(c.LogicalProcessors)) + cacheVals[x] = c + x++ + } + + return cacheVals, nil +} + +func memoryCacheLevel(paths *linuxpath.Paths, nodeID int, lpID int, cacheIndex int) int { + levelPath := filepath.Join( + paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), + "level", + ) + levelContents, err := ioutil.ReadFile(levelPath) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) + return -1 + } + // levelContents is now a []byte with the last byte being a newline + // character. Trim that off and convert the contents to an integer. + level, err := strconv.Atoi(string(levelContents[:len(levelContents)-1])) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Unable to parse int from %s\n", levelContents) + return -1 + } + return level +} + +func memoryCacheSize(paths *linuxpath.Paths, nodeID int, lpID int, cacheIndex int) int { + sizePath := filepath.Join( + paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), + "size", + ) + sizeContents, err := ioutil.ReadFile(sizePath) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) + return -1 + } + // size comes as XK\n, so we trim off the K and the newline. + size, err := strconv.Atoi(string(sizeContents[:len(sizeContents)-2])) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Unable to parse int from %s\n", sizeContents) + return -1 + } + return size +} + +func memoryCacheType(paths *linuxpath.Paths, nodeID int, lpID int, cacheIndex int) CacheType { + typePath := filepath.Join( + paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), + "type", + ) + cacheTypeContents, err := ioutil.ReadFile(typePath) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) + return CACHE_TYPE_UNIFIED + } + switch string(cacheTypeContents[:len(cacheTypeContents)-1]) { + case "Data": + return CACHE_TYPE_DATA + case "Instruction": + return CACHE_TYPE_INSTRUCTION + default: + return CACHE_TYPE_UNIFIED + } +} + +func memoryCacheSharedCPUMap(paths *linuxpath.Paths, nodeID int, lpID int, cacheIndex int) string { + scpuPath := filepath.Join( + paths.NodeCPUCacheIndex(nodeID, lpID, cacheIndex), + "shared_cpu_map", + ) + sharedCpuMap, err := ioutil.ReadFile(scpuPath) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%s\n", err) + return "" + } + return string(sharedCpuMap[:len(sharedCpuMap)-1]) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/memory/memory_linux.go b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_linux.go new file mode 100644 index 0000000000..2fb85b71d6 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_linux.go @@ -0,0 +1,237 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package memory + +import ( + "bufio" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/linuxpath" + "github.com/jaypipes/ghw/pkg/unitutil" + "github.com/jaypipes/ghw/pkg/util" +) + +const ( + _WARN_CANNOT_DETERMINE_PHYSICAL_MEMORY = ` +Could not determine total physical bytes of memory. This may +be due to the host being a virtual machine or container with no +/var/log/syslog file or /sys/devices/system/memory directory, or +the current user may not have necessary privileges to read the syslog. +We are falling back to setting the total physical amount of memory to +the total usable amount of memory +` +) + +var ( + // System log lines will look similar to the following: + // ... kernel: [0.000000] Memory: 24633272K/25155024K ... + _REGEX_SYSLOG_MEMLINE = regexp.MustCompile(`Memory:\s+\d+K\/(\d+)K`) +) + +func (i *Info) load() error { + paths := linuxpath.New(i.ctx) + tub := memTotalUsableBytes(paths) + if tub < 1 { + return fmt.Errorf("Could not determine total usable bytes of memory") + } + i.TotalUsableBytes = tub + tpb := memTotalPhysicalBytes(paths) + i.TotalPhysicalBytes = tpb + if tpb < 1 { + i.ctx.Warn(_WARN_CANNOT_DETERMINE_PHYSICAL_MEMORY) + i.TotalPhysicalBytes = tub + } + i.SupportedPageSizes = memSupportedPageSizes(paths) + return nil +} + +func memTotalPhysicalBytes(paths *linuxpath.Paths) (total int64) { + defer func() { + // fallback to the syslog file approach in case of error + if total < 0 { + total = memTotalPhysicalBytesFromSyslog(paths) + } + }() + + // detect physical memory from /sys/devices/system/memory + dir := paths.SysDevicesSystemMemory + + // get the memory block size in byte in hexadecimal notation + blockSize := filepath.Join(dir, "block_size_bytes") + + d, err := ioutil.ReadFile(blockSize) + if err != nil { + return -1 + } + blockSizeBytes, err := strconv.ParseUint(strings.TrimSpace(string(d)), 16, 64) + if err != nil { + return -1 + } + + // iterate over memory's block /sys/devices/system/memory/memory*, + // if the memory block state is 'online' we increment the total + // with the memory block size to determine the amount of physical + // memory available on this system + sysMemory, err := filepath.Glob(filepath.Join(dir, "memory*")) + if err != nil { + return -1 + } else if sysMemory == nil { + return -1 + } + + for _, path := range sysMemory { + s, err := ioutil.ReadFile(filepath.Join(path, "state")) + if err != nil { + return -1 + } + if strings.TrimSpace(string(s)) != "online" { + continue + } + total += int64(blockSizeBytes) + } + + return total +} + +func memTotalPhysicalBytesFromSyslog(paths *linuxpath.Paths) int64 { + // In Linux, the total physical memory can be determined by looking at the + // output of dmidecode, however dmidecode requires root privileges to run, + // so instead we examine the system logs for startup information containing + // total physical memory and cache the results of this. + findPhysicalKb := func(line string) int64 { + matches := _REGEX_SYSLOG_MEMLINE.FindStringSubmatch(line) + if len(matches) == 2 { + i, err := strconv.Atoi(matches[1]) + if err != nil { + return -1 + } + return int64(i * 1024) + } + return -1 + } + + // /var/log will contain a file called syslog and 0 or more files called + // syslog.$NUMBER or syslog.$NUMBER.gz containing system log records. We + // search each, stopping when we match a system log record line that + // contains physical memory information. + logDir := paths.VarLog + logFiles, err := ioutil.ReadDir(logDir) + if err != nil { + return -1 + } + for _, file := range logFiles { + if strings.HasPrefix(file.Name(), "syslog") { + fullPath := filepath.Join(logDir, file.Name()) + unzip := strings.HasSuffix(file.Name(), ".gz") + var r io.ReadCloser + r, err = os.Open(fullPath) + if err != nil { + return -1 + } + defer util.SafeClose(r) + if unzip { + r, err = gzip.NewReader(r) + if err != nil { + return -1 + } + } + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + size := findPhysicalKb(line) + if size > 0 { + return size + } + } + } + } + return -1 +} + +func memTotalUsableBytes(paths *linuxpath.Paths) int64 { + // In Linux, /proc/meminfo contains a set of memory-related amounts, with + // lines looking like the following: + // + // $ cat /proc/meminfo + // MemTotal: 24677596 kB + // MemFree: 21244356 kB + // MemAvailable: 22085432 kB + // ... + // HugePages_Total: 0 + // HugePages_Free: 0 + // HugePages_Rsvd: 0 + // HugePages_Surp: 0 + // ... + // + // It's worth noting that /proc/meminfo returns exact information, not + // "theoretical" information. For instance, on the above system, I have + // 24GB of RAM but MemTotal is indicating only around 23GB. This is because + // MemTotal contains the exact amount of *usable* memory after accounting + // for the kernel's resident memory size and a few reserved bits. For more + // information, see: + // + // https://www.kernel.org/doc/Documentation/filesystems/proc.txt + filePath := paths.ProcMeminfo + r, err := os.Open(filePath) + if err != nil { + return -1 + } + defer util.SafeClose(r) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + key := strings.Trim(parts[0], ": \t") + if key != "MemTotal" { + continue + } + value, err := strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil { + return -1 + } + inKb := (len(parts) == 3 && strings.TrimSpace(parts[2]) == "kB") + if inKb { + value = value * int(unitutil.KB) + } + return int64(value) + } + return -1 +} + +func memSupportedPageSizes(paths *linuxpath.Paths) []uint64 { + // In Linux, /sys/kernel/mm/hugepages contains a directory per page size + // supported by the kernel. The directory name corresponds to the pattern + // 'hugepages-{pagesize}kb' + dir := paths.SysKernelMMHugepages + out := make([]uint64, 0) + + files, err := ioutil.ReadDir(dir) + if err != nil { + return out + } + for _, file := range files { + parts := strings.Split(file.Name(), "-") + sizeStr := parts[1] + // Cut off the 'kb' + sizeStr = sizeStr[0 : len(sizeStr)-2] + size, err := strconv.Atoi(sizeStr) + if err != nil { + return out + } + out = append(out, uint64(size*int(unitutil.KB))) + } + return out +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/memory/memory_stub.go b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_stub.go new file mode 100644 index 0000000000..26a28b0565 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_stub.go @@ -0,0 +1,17 @@ +// +build !linux,!windows +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package memory + +import ( + "runtime" + + "github.com/pkg/errors" +) + +func (i *Info) load() error { + return errors.New("mem.Info.load not implemented on " + runtime.GOOS) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/memory/memory_windows.go b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_windows.go new file mode 100644 index 0000000000..c3a3945ca9 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/memory/memory_windows.go @@ -0,0 +1,72 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package memory + +import ( + "github.com/StackExchange/wmi" + + "github.com/jaypipes/ghw/pkg/unitutil" +) + +const wqlOperatingSystem = "SELECT TotalVisibleMemorySize FROM Win32_OperatingSystem" + +type win32OperatingSystem struct { + TotalVisibleMemorySize *uint64 +} + +const wqlPhysicalMemory = "SELECT BankLabel, Capacity, DataWidth, Description, DeviceLocator, Manufacturer, Model, Name, PartNumber, PositionInRow, SerialNumber, Speed, Tag, TotalWidth FROM Win32_PhysicalMemory" + +type win32PhysicalMemory struct { + BankLabel *string + Capacity *uint64 + DataWidth *uint16 + Description *string + DeviceLocator *string + Manufacturer *string + Model *string + Name *string + PartNumber *string + PositionInRow *uint32 + SerialNumber *string + Speed *uint32 + Tag *string + TotalWidth *uint16 +} + +func (i *Info) load() error { + // Getting info from WMI + var win32OSDescriptions []win32OperatingSystem + if err := wmi.Query(wqlOperatingSystem, &win32OSDescriptions); err != nil { + return err + } + var win32MemDescriptions []win32PhysicalMemory + if err := wmi.Query(wqlPhysicalMemory, &win32MemDescriptions); err != nil { + return err + } + // We calculate total physical memory size by summing the DIMM sizes + var totalPhysicalBytes uint64 + i.Modules = make([]*Module, 0, len(win32MemDescriptions)) + for _, description := range win32MemDescriptions { + totalPhysicalBytes += *description.Capacity + i.Modules = append(i.Modules, &Module{ + Label: *description.BankLabel, + Location: *description.DeviceLocator, + SerialNumber: *description.SerialNumber, + SizeBytes: int64(*description.Capacity), + Vendor: *description.Manufacturer, + }) + } + var totalUsableBytes uint64 + for _, description := range win32OSDescriptions { + // TotalVisibleMemorySize is the amount of memory available for us by + // the operating system **in Kilobytes** + totalUsableBytes += *description.TotalVisibleMemorySize * uint64(unitutil.KB) + } + i.TotalUsableBytes = int64(totalUsableBytes) + i.TotalPhysicalBytes = int64(totalPhysicalBytes) + return nil +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/option/option.go b/vendor/github.com/jaypipes/ghw/pkg/option/option.go new file mode 100644 index 0000000000..0af8b4cbe6 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/option/option.go @@ -0,0 +1,227 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package option + +import ( + "io" + "io/ioutil" + "log" + "os" +) + +const ( + defaultChroot = "/" + envKeyChroot = "GHW_CHROOT" + envKeyDisableWarnings = "GHW_DISABLE_WARNINGS" + envKeyDisableTools = "GHW_DISABLE_TOOLS" + envKeySnapshotPath = "GHW_SNAPSHOT_PATH" + envKeySnapshotRoot = "GHW_SNAPSHOT_ROOT" + envKeySnapshotExclusive = "GHW_SNAPSHOT_EXCLUSIVE" + envKeySnapshotPreserve = "GHW_SNAPSHOT_PRESERVE" +) + +// Alerter emits warnings about undesirable but recoverable errors. +// We use a subset of a logger interface only to emit warnings, and +// `Warninger` sounded ugly. +type Alerter interface { + Printf(format string, v ...interface{}) +} + +var ( + NullAlerter = log.New(ioutil.Discard, "", 0) +) + +// EnvOrDefaultAlerter returns the default instance ghw will use to emit +// its warnings. ghw will emit warnings to stderr by default unless the +// environs variable GHW_DISABLE_WARNINGS is specified; in the latter case +// all warning will be suppressed. +func EnvOrDefaultAlerter() Alerter { + var dest io.Writer + if _, exists := os.LookupEnv(envKeyDisableWarnings); exists { + dest = ioutil.Discard + } else { + // default + dest = os.Stderr + } + return log.New(dest, "", 0) +} + +// EnvOrDefaultChroot returns the value of the GHW_CHROOT environs variable or +// the default value of "/" if not set +func EnvOrDefaultChroot() string { + // Grab options from the environs by default + if val, exists := os.LookupEnv(envKeyChroot); exists { + return val + } + return defaultChroot +} + +// EnvOrDefaultSnapshotPath returns the value of the GHW_SNAPSHOT_PATH environs variable +// or the default value of "" (disable snapshot consumption) if not set +func EnvOrDefaultSnapshotPath() string { + if val, exists := os.LookupEnv(envKeySnapshotPath); exists { + return val + } + return "" // default is no snapshot +} + +// EnvOrDefaultSnapshotRoot returns the value of the the GHW_SNAPSHOT_ROOT environs variable +// or the default value of "" (self-manage the snapshot unpack directory, if relevant) if not set +func EnvOrDefaultSnapshotRoot() string { + if val, exists := os.LookupEnv(envKeySnapshotRoot); exists { + return val + } + return "" // default is to self-manage the snapshot directory +} + +// EnvOrDefaultSnapshotExclusive returns the value of the GHW_SNAPSHOT_EXCLUSIVE environs variable +// or the default value of false if not set +func EnvOrDefaultSnapshotExclusive() bool { + if _, exists := os.LookupEnv(envKeySnapshotExclusive); exists { + return true + } + return false +} + +// EnvOrDefaultSnapshotPreserve returns the value of the GHW_SNAPSHOT_PRESERVE environs variable +// or the default value of false if not set +func EnvOrDefaultSnapshotPreserve() bool { + if _, exists := os.LookupEnv(envKeySnapshotPreserve); exists { + return true + } + return false +} + +// EnvOrDefaultTools return true if ghw should use external tools to augment the data collected +// from sysfs. Most users want to do this most of time, so this is enabled by default. +// Users consuming snapshots may want to opt out, thus they can set the GHW_DISABLE_TOOLS +// environs variable to any value to make ghw skip calling external tools even if they are available. +func EnvOrDefaultTools() bool { + if _, exists := os.LookupEnv(envKeyDisableTools); exists { + return false + } + return true +} + +// Option is used to represent optionally-configured settings. Each field is a +// pointer to some concrete value so that we can tell when something has been +// set or left unset. +type Option struct { + // To facilitate querying of sysfs filesystems that are bind-mounted to a + // non-default root mountpoint, we allow users to set the GHW_CHROOT environ + // vairable to an alternate mountpoint. For instance, assume that the user of + // ghw is a Golang binary being executed from an application container that has + // certain host filesystems bind-mounted into the container at /host. The user + // would ensure the GHW_CHROOT environ variable is set to "/host" and ghw will + // build its paths from that location instead of / + Chroot *string + + // Snapshot contains options for handling ghw snapshots + Snapshot *SnapshotOptions + + // Alerter contains the target for ghw warnings + Alerter Alerter + + // EnableTools optionally request ghw to not call any external program to learn + // about the hardware. The default is to use such tools if available. + EnableTools *bool +} + +// SnapshotOptions contains options for handling of ghw snapshots +type SnapshotOptions struct { + // Path allows users to specify a snapshot (captured using ghw-snapshot) to be + // automatically consumed. Users need to supply the path of the snapshot, and + // ghw will take care of unpacking it on a temporary directory. + // Set the environment variable "GHW_SNAPSHOT_PRESERVE" to make ghw skip the cleanup + // stage and keep the unpacked snapshot in the temporary directory. + Path string + // Root is the directory on which the snapshot must be unpacked. This allows + // the users to manage their snapshot directory instead of ghw doing that on + // their behalf. Relevant only if SnapshotPath is given. + Root *string + // Exclusive tells ghw if the given directory should be considered of exclusive + // usage of ghw or not If the user provides a Root. If the flag is set, ghw will + // unpack the snapshot in the given SnapshotRoot iff the directory is empty; otherwise + // any existing content will be left untouched and the unpack stage will exit silently. + // As additional side effect, give both this option and SnapshotRoot to make each + // context try to unpack the snapshot only once. + Exclusive bool +} + +func WithChroot(dir string) *Option { + return &Option{Chroot: &dir} +} + +// WithSnapshot sets snapshot-processing options for a ghw run +func WithSnapshot(opts SnapshotOptions) *Option { + return &Option{ + Snapshot: &opts, + } +} + +// WithAlerter sets alerting options for ghw +func WithAlerter(alerter Alerter) *Option { + return &Option{ + Alerter: alerter, + } +} + +// WithNullAlerter sets No-op alerting options for ghw +func WithNullAlerter() *Option { + return &Option{ + Alerter: NullAlerter, + } +} + +// WithDisableTools sets enables or prohibts ghw to call external tools to discover hardware capabilities. +func WithDisableTools() *Option { + false_ := false + return &Option{EnableTools: &false_} +} + +// There is intentionally no Option related to GHW_SNAPSHOT_PRESERVE because we see that as +// a debug/troubleshoot aid more something users wants to do regularly. +// Hence we allow that only via the environment variable for the time being. + +func Merge(opts ...*Option) *Option { + merged := &Option{} + for _, opt := range opts { + if opt.Chroot != nil { + merged.Chroot = opt.Chroot + } + if opt.Snapshot != nil { + merged.Snapshot = opt.Snapshot + } + if opt.Alerter != nil { + merged.Alerter = opt.Alerter + } + if opt.EnableTools != nil { + merged.EnableTools = opt.EnableTools + } + } + // Set the default value if missing from mergeOpts + if merged.Chroot == nil { + chroot := EnvOrDefaultChroot() + merged.Chroot = &chroot + } + if merged.Alerter == nil { + merged.Alerter = EnvOrDefaultAlerter() + } + if merged.Snapshot == nil { + snapRoot := EnvOrDefaultSnapshotRoot() + merged.Snapshot = &SnapshotOptions{ + Path: EnvOrDefaultSnapshotPath(), + Root: &snapRoot, + Exclusive: EnvOrDefaultSnapshotExclusive(), + } + } + if merged.EnableTools == nil { + enabled := EnvOrDefaultTools() + merged.EnableTools = &enabled + } + return merged +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/pci/address/address.go b/vendor/github.com/jaypipes/ghw/pkg/pci/address/address.go new file mode 100644 index 0000000000..7b1360536e --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/pci/address/address.go @@ -0,0 +1,55 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package address + +import ( + "regexp" + "strings" +) + +var ( + regexAddress *regexp.Regexp = regexp.MustCompile( + `^(([0-9a-f]{0,4}):)?([0-9a-f]{2}):([0-9a-f]{2})\.([0-9a-f]{1})$`, + ) +) + +type Address struct { + Domain string + Bus string + Slot string + Function string +} + +// String() returns the canonical [D]BSF representation of this Address +func (addr *Address) String() string { + return addr.Domain + ":" + addr.Bus + ":" + addr.Slot + "." + addr.Function +} + +// Given a string address, returns a complete Address struct, filled in with +// domain, bus, slot and function components. The address string may either +// be in $BUS:$SLOT.$FUNCTION (BSF) format or it can be a full PCI address +// that includes the 4-digit $DOMAIN information as well: +// $DOMAIN:$BUS:$SLOT.$FUNCTION. +// +// Returns "" if the address string wasn't a valid PCI address. +func FromString(address string) *Address { + addrLowered := strings.ToLower(address) + matches := regexAddress.FindStringSubmatch(addrLowered) + if len(matches) == 6 { + dom := "0000" + if matches[1] != "" { + dom = matches[2] + } + return &Address{ + Domain: dom, + Bus: matches[3], + Slot: matches[4], + Function: matches[5], + } + } + return nil +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree.go b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree.go new file mode 100644 index 0000000000..d2f75116aa --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree.go @@ -0,0 +1,263 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package snapshot + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// Attempting to tar up pseudofiles like /proc/cpuinfo is an exercise in +// futility. Notably, the pseudofiles, when read by syscalls, do not return the +// number of bytes read. This causes the tar writer to write zero-length files. +// +// Instead, it is necessary to build a directory structure in a tmpdir and +// create actual files with copies of the pseudofile contents + +// CloneTreeInto copies all the pseudofiles that ghw will consume into the root +// `scratchDir`, preserving the hieratchy. +func CloneTreeInto(scratchDir string) error { + var err error + + var createPaths = []string{ + "sys/block", + } + + for _, path := range createPaths { + if err = os.MkdirAll(filepath.Join(scratchDir, path), os.ModePerm); err != nil { + return err + } + } + + if err = createBlockDevices(scratchDir); err != nil { + return err + } + + fileSpecs := ExpectedCloneContent() + return CopyFilesInto(fileSpecs, scratchDir, nil) +} + +// ExpectedCloneContent return a slice of glob patterns which represent the pseudofiles +// ghw cares about. +// The intended usage of this function is to validate a clone tree, checking that the +// content matches the expectations. +// Beware: the content is host-specific, because the content pertaining some subsystems, +// most notably PCI, is host-specific and unpredictable. +func ExpectedCloneContent() []string { + fileSpecs := ExpectedCloneStaticContent() + fileSpecs = append(fileSpecs, ExpectedCloneNetContent()...) + fileSpecs = append(fileSpecs, ExpectedClonePCIContent()...) + fileSpecs = append(fileSpecs, ExpectedCloneGPUContent()...) + return fileSpecs +} + +// ExpectedCloneStaticContent return a slice of glob patterns which represent the pseudofiles +// ghw cares about, and which are independent from host specific topology or configuration, +// thus are safely represented by a static slice - e.g. they don't need to be discovered at runtime. +func ExpectedCloneStaticContent() []string { + return []string{ + "/etc/mtab", + "/proc/cpuinfo", + "/proc/meminfo", + "/sys/devices/system/cpu/cpu*/cache/index*/*", + "/sys/devices/system/cpu/cpu*/topology/*", + "/sys/devices/system/memory/block_size_bytes", + "/sys/devices/system/memory/memory*/online", + "/sys/devices/system/memory/memory*/state", + "/sys/devices/system/node/has_*", + "/sys/devices/system/node/online", + "/sys/devices/system/node/possible", + "/sys/devices/system/node/node*/cpu*", + "/sys/devices/system/node/node*/distance", + } +} + +// ValidateClonedTree checks the content of a cloned tree, whose root is `clonedDir`, +// against a slice of glob specs which must be included in the cloned tree. +// Is not wrong, and this functions doesn't enforce this, that the cloned tree includes +// more files than the necessary; ghw will just ignore the files it doesn't care about. +// Returns a slice of glob patters expected (given) but not found in the cloned tree, +// and the error during the validation (if any). +func ValidateClonedTree(fileSpecs []string, clonedDir string) ([]string, error) { + missing := []string{} + for _, fileSpec := range fileSpecs { + matches, err := filepath.Glob(filepath.Join(clonedDir, fileSpec)) + if err != nil { + return missing, err + } + if len(matches) == 0 { + missing = append(missing, fileSpec) + } + } + return missing, nil +} + +func copyPseudoFile(path, targetPath string) error { + buf, err := ioutil.ReadFile(path) + if err != nil { + return err + } + trace("creating %s\n", targetPath) + f, err := os.Create(targetPath) + if err != nil { + return err + } + if _, err = f.Write(buf); err != nil { + return err + } + f.Close() + return nil +} + +// CopyFileOptions allows to finetune the behaviour of the CopyFilesInto function +type CopyFileOptions struct { + // IsSymlinkFn allows to control the behaviour when handling a symlink. + // If this hook returns true, the source file is treated as symlink: the cloned + // tree will thus contain a symlink, with its path adjusted to match the relative + // path inside the cloned tree. If return false, the symlink will be deferred. + // The easiest use case of this hook is if you want to avoid symlinks in your cloned + // tree (having duplicated content). In this case you can just add a function + // which always return false. + IsSymlinkFn func(path string, info os.FileInfo) bool +} + +// CopyFilesInto copies all the given glob files specs in the given `destDir` directory, +// preserving the directory structure. This means you can provide a deeply nested filespec +// like +// - /some/deeply/nested/file* +// and you DO NOT need to build the tree incrementally like +// - /some/ +// - /some/deeply/ +// ... +// all glob patterns supported in `filepath.Glob` are supported. +func CopyFilesInto(fileSpecs []string, destDir string, opts *CopyFileOptions) error { + if opts == nil { + opts = &CopyFileOptions{ + IsSymlinkFn: isSymlink, + } + } + for _, fileSpec := range fileSpecs { + trace("copying spec: %q\n", fileSpec) + matches, err := filepath.Glob(fileSpec) + if err != nil { + return err + } + if err := copyFileTreeInto(matches, destDir, opts); err != nil { + return err + } + } + return nil +} + +func copyFileTreeInto(paths []string, destDir string, opts *CopyFileOptions) error { + for _, path := range paths { + trace(" copying path: %q\n", path) + baseDir := filepath.Dir(path) + if err := os.MkdirAll(filepath.Join(destDir, baseDir), os.ModePerm); err != nil { + return err + } + + fi, err := os.Lstat(path) + if err != nil { + return err + } + // directories must be listed explicitely and created separately. + // In the future we may want to expose this decision as hook point in + // CopyFileOptions, when clear use cases emerge. + if fi.IsDir() { + trace("expanded glob path %q is a directory - skipped", path) + continue + } + if opts.IsSymlinkFn(path, fi) { + trace(" copying link: %q\n", path) + if err := copyLink(path, filepath.Join(destDir, path)); err != nil { + return err + } + } else { + trace(" copying file: %q\n", path) + if err := copyPseudoFile(path, filepath.Join(destDir, path)); err != nil { + return err + } + } + } + return nil +} + +func isSymlink(path string, fi os.FileInfo) bool { + return fi.Mode()&os.ModeSymlink != 0 +} + +func copyLink(path, targetPath string) error { + target, err := os.Readlink(path) + if err != nil { + return err + } + if err := os.Symlink(target, targetPath); err != nil { + return err + } + + return nil +} + +type filterFunc func(string) bool + +// cloneContentByClass copies all the content related to a given device class +// (devClass), possibly filtering out devices whose name does NOT pass a +// filter (filterName). Each entry in `/sys/class/$CLASS` is actually a +// symbolic link. We can filter out entries depending on the link target. +// Each filter is a simple function which takes the entry name or the link +// target and must return true if the entry should be collected, false +// otherwise. Last, explicitely collect a list of attributes for each entry, +// given as list of glob patterns as `subEntries`. +// Return the final list of glob patterns to be collected. +func cloneContentByClass(devClass string, subEntries []string, filterName filterFunc, filterLink filterFunc) []string { + var fileSpecs []string + + // warning: don't use the context package here, this means not even the linuxpath package. + // TODO(fromani) remove the path duplication + sysClass := filepath.Join("sys", "class", devClass) + entries, err := ioutil.ReadDir(sysClass) + if err != nil { + // we should not import context, hence we can't Warn() + return fileSpecs + } + for _, entry := range entries { + devName := entry.Name() + + if !filterName(devName) { + continue + } + + devPath := filepath.Join(sysClass, devName) + dest, err := os.Readlink(devPath) + if err != nil { + continue + } + + if !filterLink(dest) { + continue + } + + // so, first copy the symlink itself + fileSpecs = append(fileSpecs, devPath) + // now we have to clone the content of the actual entry + // related (and found into a subdir of) the backing hardware + // device + devData := filepath.Clean(filepath.Join(sysClass, dest)) + for _, subEntry := range subEntries { + fileSpecs = append(fileSpecs, filepath.Join(devData, subEntry)) + } + } + + return fileSpecs +} + +// filterNone allows all content, filtering out none of it +func filterNone(_ string) bool { + return true +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_block.go b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_block.go new file mode 100644 index 0000000000..18e2161a4e --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_block.go @@ -0,0 +1,221 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package snapshot + +import ( + "errors" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +func createBlockDevices(buildDir string) error { + // Grab all the block device pseudo-directories from /sys/block symlinks + // (excluding loopback devices) and inject them into our build filesystem + // with all but the circular symlink'd subsystem directories + devLinks, err := ioutil.ReadDir("/sys/block") + if err != nil { + return err + } + for _, devLink := range devLinks { + dname := devLink.Name() + if strings.HasPrefix(dname, "loop") { + continue + } + devPath := filepath.Join("/sys/block", dname) + trace("processing block device %q\n", devPath) + + // from the sysfs layout, we know this is always a symlink + linkContentPath, err := os.Readlink(devPath) + if err != nil { + return err + } + trace("link target for block device %q is %q\n", devPath, linkContentPath) + + // Create a symlink in our build filesystem that is a directory + // pointing to the actual device bus path where the block device's + // information directory resides + linkPath := filepath.Join(buildDir, "sys/block", dname) + linkTargetPath := filepath.Join( + buildDir, + "sys/block", + strings.TrimPrefix(linkContentPath, string(os.PathSeparator)), + ) + trace("creating device directory %s\n", linkTargetPath) + if err = os.MkdirAll(linkTargetPath, os.ModePerm); err != nil { + return err + } + + trace("linking device directory %s to %s\n", linkPath, linkContentPath) + // Make sure the link target is a relative path! + // if we use absolute path, the link target will be an absolute path starting + // with buildDir, hence the snapshot will contain broken link. + // Otherwise, the unpack directory will never have the same prefix of buildDir! + if err = os.Symlink(linkContentPath, linkPath); err != nil { + return err + } + // Now read the source block device directory and populate the + // newly-created target link in the build directory with the + // appropriate block device pseudofiles + srcDeviceDir := filepath.Join( + "/sys/block", + strings.TrimPrefix(linkContentPath, string(os.PathSeparator)), + ) + trace("creating device directory %q from %q\n", linkTargetPath, srcDeviceDir) + if err = createBlockDeviceDir(linkTargetPath, srcDeviceDir); err != nil { + return err + } + } + return nil +} + +func createBlockDeviceDir(buildDeviceDir string, srcDeviceDir string) error { + // Populate the supplied directory (in our build filesystem) with all the + // appropriate information pseudofile contents for the block device. + devName := filepath.Base(srcDeviceDir) + devFiles, err := ioutil.ReadDir(srcDeviceDir) + if err != nil { + return err + } + for _, f := range devFiles { + fname := f.Name() + fp := filepath.Join(srcDeviceDir, fname) + fi, err := os.Lstat(fp) + if err != nil { + return err + } + if fi.Mode()&os.ModeSymlink != 0 { + // Ignore any symlinks in the deviceDir since they simply point to + // either self-referential links or information we aren't + // interested in like "subsystem" + continue + } else if fi.IsDir() { + if strings.HasPrefix(fname, devName) { + // We're interested in are the directories that begin with the + // block device name. These are directories with information + // about the partitions on the device + buildPartitionDir := filepath.Join( + buildDeviceDir, fname, + ) + srcPartitionDir := filepath.Join( + srcDeviceDir, fname, + ) + trace("creating partition directory %s\n", buildPartitionDir) + err = os.MkdirAll(buildPartitionDir, os.ModePerm) + if err != nil { + return err + } + err = createPartitionDir(buildPartitionDir, srcPartitionDir) + if err != nil { + return err + } + } + } else if fi.Mode().IsRegular() { + // Regular files in the block device directory are both regular and + // pseudofiles containing information such as the size (in sectors) + // and whether the device is read-only + buf, err := ioutil.ReadFile(fp) + if err != nil { + if errors.Is(err, os.ErrPermission) { + // example: /sys/devices/virtual/block/zram0/compact is 0400 + trace("permission denied reading %q - skipped\n", fp) + continue + } + return err + } + targetPath := filepath.Join(buildDeviceDir, fname) + trace("creating %s\n", targetPath) + f, err := os.Create(targetPath) + if err != nil { + return err + } + if _, err = f.Write(buf); err != nil { + return err + } + f.Close() + } + } + // There is a special file $DEVICE_DIR/queue/rotational that, for some hard + // drives, contains a 1 or 0 indicating whether the device is a spinning + // disk or not + srcQueueDir := filepath.Join( + srcDeviceDir, + "queue", + ) + buildQueueDir := filepath.Join( + buildDeviceDir, + "queue", + ) + err = os.MkdirAll(buildQueueDir, os.ModePerm) + if err != nil { + return err + } + fp := filepath.Join(srcQueueDir, "rotational") + buf, err := ioutil.ReadFile(fp) + if err != nil { + return err + } + targetPath := filepath.Join(buildQueueDir, "rotational") + trace("creating %s\n", targetPath) + f, err := os.Create(targetPath) + if err != nil { + return err + } + if _, err = f.Write(buf); err != nil { + return err + } + f.Close() + + return nil +} + +func createPartitionDir(buildPartitionDir string, srcPartitionDir string) error { + // Populate the supplied directory (in our build filesystem) with all the + // appropriate information pseudofile contents for the partition. + partFiles, err := ioutil.ReadDir(srcPartitionDir) + if err != nil { + return err + } + for _, f := range partFiles { + fname := f.Name() + fp := filepath.Join(srcPartitionDir, fname) + fi, err := os.Lstat(fp) + if err != nil { + return err + } + if fi.Mode()&os.ModeSymlink != 0 { + // Ignore any symlinks in the partition directory since they simply + // point to information we aren't interested in like "subsystem" + continue + } else if fi.IsDir() { + // The subdirectories in the partition directory are not + // interesting for us. They have information about power events and + // traces + continue + } else if fi.Mode().IsRegular() { + // Regular files in the block device directory are both regular and + // pseudofiles containing information such as the size (in sectors) + // and whether the device is read-only + buf, err := ioutil.ReadFile(fp) + if err != nil { + return err + } + targetPath := filepath.Join(buildPartitionDir, fname) + trace("creating %s\n", targetPath) + f, err := os.Create(targetPath) + if err != nil { + return err + } + if _, err = f.Write(buf); err != nil { + return err + } + f.Close() + } + } + return nil +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_gpu.go b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_gpu.go new file mode 100644 index 0000000000..a26d6b01fb --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_gpu.go @@ -0,0 +1,33 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package snapshot + +import ( + "strings" +) + +// ExpectedCloneGPUContent returns a slice of strings pertaining to the GPU devices ghw +// cares about. We cannot use a static list because we want to grab only the first cardX data +// (see comment in pkg/gpu/gpu_linux.go) +// Additionally, we want to make sure to clone the backing device data. +func ExpectedCloneGPUContent() []string { + cardEntries := []string{ + "device", + } + + filterName := func(cardName string) bool { + if !strings.HasPrefix(cardName, "card") { + return false + } + if strings.ContainsRune(cardName, '-') { + return false + } + return true + } + + return cloneContentByClass("drm", cardEntries, filterName, filterNone) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_net.go b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_net.go new file mode 100644 index 0000000000..6c89a6cc39 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_net.go @@ -0,0 +1,31 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package snapshot + +import ( + "strings" +) + +// ExpectedCloneNetContent returns a slice of strings pertaning to the network interfaces ghw +// cares about. We cannot use a static list because we want to filter away the virtual devices, +// which ghw doesn't concern itself about. So we need to do some runtime discovery. +// Additionally, we want to make sure to clone the backing device data. +func ExpectedCloneNetContent() []string { + ifaceEntries := []string{ + "addr_assign_type", + // intentionally avoid to clone "address" to avoid to leak any host-idenfifiable data. + } + + filterLink := func(linkDest string) bool { + if strings.Contains(linkDest, "devices/virtual/net") { + return false + } + return true + } + + return cloneContentByClass("net", ifaceEntries, filterNone, filterLink) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_pci.go b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_pci.go new file mode 100644 index 0000000000..503a562d9c --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/snapshot/clonetree_pci.go @@ -0,0 +1,148 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package snapshot + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + pciaddr "github.com/jaypipes/ghw/pkg/pci/address" +) + +const ( + // root directory: entry point to start scanning the PCI forest + // warning: don't use the context package here, this means not even the linuxpath package. + // TODO(fromani) remove the path duplication + sysBusPCIDir = "/sys/bus/pci/devices" +) + +// ExpectedClonePCIContent return a slice of glob patterns which represent the pseudofiles +// ghw cares about, pertaining to PCI devices only. +// Beware: the content is host-specific, because the PCI topology is host-dependent and unpredictable. +func ExpectedClonePCIContent() []string { + var fileSpecs []string + pciRoots := []string{ + sysBusPCIDir, + } + for { + if len(pciRoots) == 0 { + break + } + pciRoot := pciRoots[0] + pciRoots = pciRoots[1:] + specs, roots := scanPCIDeviceRoot(pciRoot) + pciRoots = append(pciRoots, roots...) + fileSpecs = append(fileSpecs, specs...) + } + return fileSpecs +} + +// scanPCIDeviceRoot reports a slice of glob patterns which represent the pseudofiles +// ghw cares about pertaining to all the PCI devices connected to the bus connected from the +// given root; usually (but not always) a CPU packages has 1+ PCI(e) roots, forming the first +// level; more PCI bridges are (usually) attached to this level, creating deep nested trees. +// hence we need to scan all possible roots, to make sure not to miss important devices. +// +// note about notifying errors. This function and its helper functions do use trace() everywhere +// to report recoverable errors, even though it would have been appropriate to use Warn(). +// This is unfortunate, and again a byproduct of the fact we cannot use context.Context to avoid +// circular dependencies. +// TODO(fromani): switch to Warn() as soon as we figure out how to break this circular dep. +func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { + trace("scanning PCI device root %q\n", root) + + perDevEntries := []string{ + "class", + "device", + "irq", + "local_cpulist", + "modalias", + "numa_node", + "revision", + "vendor", + } + entries, err := ioutil.ReadDir(root) + if err != nil { + return []string{}, []string{} + } + for _, entry := range entries { + entryName := entry.Name() + if addr := pciaddr.FromString(entryName); addr == nil { + // doesn't look like a entry we care about + // This is by far and large the most likely path + // hence we should NOT trace/warn here. + continue + } + + entryPath := filepath.Join(root, entryName) + pciEntry, err := findPCIEntryFromPath(root, entryName) + if err != nil { + trace("error scanning %q: %v", entryName, err) + continue + } + + trace("PCI entry is %q\n", pciEntry) + fileSpecs = append(fileSpecs, entryPath) + for _, perNetEntry := range perDevEntries { + fileSpecs = append(fileSpecs, filepath.Join(pciEntry, perNetEntry)) + } + + if isPCIBridge(entryPath) { + trace("adding new PCI root %q\n", entryName) + pciRoots = append(pciRoots, pciEntry) + } + } + return fileSpecs, pciRoots +} + +func findPCIEntryFromPath(root, entryName string) (string, error) { + entryPath := filepath.Join(root, entryName) + fi, err := os.Lstat(entryPath) + if err != nil { + return "", fmt.Errorf("stat(%s) failed: %v\n", entryPath, err) + } + if fi.Mode()&os.ModeSymlink == 0 { + // regular file, nothing to resolve + return entryPath, nil + } + // resolve symlink + target, err := os.Readlink(entryPath) + trace("entry %q is symlink resolved to %q\n", entryPath, target) + if err != nil { + return "", fmt.Errorf("readlink(%s) failed: %v - skipped\n", entryPath, err) + } + return filepath.Clean(filepath.Join(root, target)), nil +} + +func isPCIBridge(entryPath string) bool { + subNodes, err := ioutil.ReadDir(entryPath) + if err != nil { + // this is so unlikely we don't even return error. But we trace just in case. + trace("error scanning device entry path %q: %v", entryPath, err) + return false + } + for _, subNode := range subNodes { + if !subNode.IsDir() { + continue + } + if addr := pciaddr.FromString(subNode.Name()); addr != nil { + // we got an entry in the directory pertaining to this device + // which is a directory itself and it is named like a PCI address. + // Hence we infer the device we are considering is a PCI bridge of sorts. + // This is is indeed a bit brutal, but the only possible alternative + // (besides blindly copying everything in /sys/bus/pci/devices) is + // to detect the type of the device and pick only the bridges. + // This approach duplicates the logic within the `pci` subkpg + // - or forces us into awkward dep cycles, and has poorer forward + // compatibility. + return true + } + } + return false +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/snapshot/pack.go b/vendor/github.com/jaypipes/ghw/pkg/snapshot/pack.go new file mode 100644 index 0000000000..8d9bd95914 --- /dev/null +++ b/vendor/github.com/jaypipes/ghw/pkg/snapshot/pack.go @@ -0,0 +1,112 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package snapshot + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// PackFrom creates the snapshot named `snapshotName` from the +// directory tree whose root is `sourceRoot`. +func PackFrom(snapshotName, sourceRoot string) error { + f, err := OpenDestination(snapshotName) + if err != nil { + return err + } + defer f.Close() + + return PackWithWriter(f, sourceRoot) +} + +// OpenDestination opens the `snapshotName` file for writing, bailing out +// if the file seems to exist and have existing content already. +// This is done to avoid accidental overwrites. +func OpenDestination(snapshotName string) (*os.File, error) { + var f *os.File + var err error + + if _, err = os.Stat(snapshotName); errors.Is(err, os.ErrNotExist) { + if f, err = os.Create(snapshotName); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } else { + f, err := os.OpenFile(snapshotName, os.O_WRONLY, 0600) + if err != nil { + return nil, err + } + fs, err := f.Stat() + if err != nil { + return nil, err + } + if fs.Size() > 0 { + return nil, fmt.Errorf("File %s already exists and is of size >0", snapshotName) + } + } + return f, nil +} + +// PakcWithWriter creates a snapshot sending all the binary data to the +// given `fw` writer. The snapshot is made from the directory tree whose +// root is `sourceRoot`. +func PackWithWriter(fw io.Writer, sourceRoot string) error { + gzw := gzip.NewWriter(fw) + defer gzw.Close() + + tw := tar.NewWriter(gzw) + defer tw.Close() + + return createSnapshot(tw, sourceRoot) +} + +func createSnapshot(tw *tar.Writer, buildDir string) error { + return filepath.Walk(buildDir, func(path string, fi os.FileInfo, err error) error { + if path == buildDir { + return nil + } + var link string + + if fi.Mode()&os.ModeSymlink != 0 { + trace("processing symlink %s\n", path) + link, err = os.Readlink(path) + if err != nil { + return err + } + } + + hdr, err := tar.FileInfoHeader(fi, link) + if err != nil { + return err + } + hdr.Name = strings.TrimPrefix(strings.TrimPrefix(path, buildDir), string(os.PathSeparator)) + + if err = tw.WriteHeader(hdr); err != nil { + return err + } + + switch hdr.Typeflag { + case tar.TypeReg, tar.TypeRegA: + f, err := os.Open(path) + if err != nil { + return err + } + if _, err = io.Copy(tw, f); err != nil { + return err + } + f.Close() + } + return nil + }) +} diff --git a/vendor/github.com/jaypipes/ghw/pkg/snapshot/testdata.tar.gz b/vendor/github.com/jaypipes/ghw/pkg/snapshot/testdata.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..edb26fbda33391756dfa941509fc5313d489f76a GIT binary patch literal 485 zcmVKX6UY9kwC`hFb3r0+H*{*{{pb2`uf7#{ zfjG&_qA-oAI&Yr@?)$%>QeyuP;k19-Z@b2r#n*vLf6my@Kb0&+{}6ubzw|^f_Sg6I z9|k|lsiKNRJdTst_ke=_*ZZ?&uHn?*KL57Axa_~s?VMYEf4~1;%WeMGIQ8$VZT68>JICV!_x>Wq z=YJTF_ILTZ;CB9F&;L`Q@cW-2p6Wjx9*94$;dK5Z?DMaN2>|CmA