fanhuadesenlinnn/kube-vip
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/kube-vip/kube-vip.git synced 2026-09-20 08:03:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update to automated testing and clear msg

Browse Source
This commit is contained in:
thebsdbox
2022-04-04 14:49:38 +01:00
parent 7b0bd4061f
commit d7b49a0f0f
7 changed files with 201 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kube-vip.go
View File

@@ -197,6 +197,7 @@ var kubeVipManager = &cobra.Command{
Short: "Start the kube-vip manager", Short: "Start the kube-vip manager",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
log.Infof("Starting kube-vip.io [%s]", Release.Version)
// parse environment variables, these will overwrite anything loaded or flags // parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig) err := kubevip.ParseEnvironment(&initConfig)
if err != nil { if err != nil {

2
pkg/cluster/service.go
View File

@@ -117,7 +117,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
if ndp != nil { if ndp != nil {
defer ndp.Close() defer ndp.Close()
} }
log.Infof("Gratuitous Arp broadcast will repeat every 3 seconds for [%s]", ipString)
for { for {
select { select {
case <-ctx.Done(): // if cancel() execute case <-ctx.Done(): // if cancel() execute

3
pkg/vip/arp.go
View File

@@ -168,7 +168,8 @@ func ARPSendGratuitous(address, ifaceName string) error {
return fmt.Errorf("failed to parse address %s", ip) return fmt.Errorf("failed to parse address %s", ip)
} }
log.Infof("Broadcasting ARP update for %s (%s) via %s", address, iface.HardwareAddr, iface.Name) // This is a debug message, enable debugging to ensure that the gratuitous arp is repeating
log.Debugf("Broadcasting ARP update for %s (%s) via %s", address, iface.HardwareAddr, iface.Name)
m, err := gratuitousARP(ip, iface.HardwareAddr) m, err := gratuitousARP(ip, iface.HardwareAddr)
if err != nil { if err != nil {
return err return err

48
testing/k3s/create.sh
View File

@@ -1,4 +1,20 @@
#!/bin/bash #!/bin/bash
set -e
# Read node configuration
source ./testing/nodes
# Read logging function
source ./testing/logging.bash
## Main()
# Ensure we have an entirely new logfile
reset_logfile
logr "INFO" "Starting kube-vip.io testing with k3s"
logr "DEFAULT" "Creating Logfile $logfile"
if [[ -z $1 && -z $2 && -z $3 && -z $4 ]]; then if [[ -z $1 && -z $2 && -z $3 && -z $4 ]]; then
echo "Usage:" echo "Usage:"
@@ -10,34 +26,34 @@ if [[ -z $1 && -z $2 && -z $3 && -z $4 ]]; then
exit 1 exit 1
fi fi
case "$2" in # Sane variable renaming
kubernetes_version=$4
kube_vip_version=$1
kube_vip_vip=$3
"controlplane") echo "Creating control plane only cluster" case "$2" in
mode="--controlplane" "controlplane") logr "INFO" "Creating in control plane only mode"
kube_vip_mode="--controlplane"
;; ;;
"services") echo "Creating services only cluster" "services") logr "INFO" "Creating in services-only mode"
mode="--services" kube_vip_mode="--services"
;; ;;
"hybrid") echo "Creating hybrid cluster" "hybrid") logr "INFO" "Creating in hybrid mode"
mode="--controlplane --services" kube_vip_mode="--controlplane --services"
;; ;;
*) echo "Unknown kube-vip mode [$2]" *) echo "Unknown kube-vip mode [$3]"
exit -1 exit -1
;; ;;
esac esac
source ./testing/nodes
echo "Creating First node!"
ssh $NODE01 "sudo mkdir -p /var/lib/rancher/k3s/server/manifests/" ssh $NODE01 "sudo mkdir -p /var/lib/rancher/k3s/server/manifests/"
ssh $NODE01 "sudo docker run --network host --rm plndr/kube-vip:$1 manifest daemonset $mode --interface ens160 --vip $3 --arp --leaderElection --inCluster --taint | sudo tee /var/lib/rancher/k3s/server/manifests/vip.yaml" ssh $NODE01 "sudo docker run --network host --rm plndr/kube-vip:$kube_vip_version manifest daemonset $kube_vip_mode --interface ens160 --vip $kube_vip_vip --arp --leaderElection --inCluster --taint | sudo tee /var/lib/rancher/k3s/server/manifests/vip.yaml"
ssh $NODE01 "sudo curl https://kube-vip.io/manifests/rbac.yaml | sudo tee /var/lib/rancher/k3s/server/manifests/rbac.yaml" ssh $NODE01 "sudo curl https://kube-vip.io/manifests/rbac.yaml | sudo tee /var/lib/rancher/k3s/server/manifests/rbac.yaml"
ssh $NODE01 "sudo screen -dmSL k3s k3s server --cluster-init --tls-san $3 --no-deploy servicelb --disable-cloud-controller --token=test" ssh $NODE01 "sudo screen -dmSL k3s k3s server --cluster-init --tls-san $kube_vip_vip --no-deploy servicelb --disable-cloud-controller --token=test"
echo "Started first node, sleeping for 60 seconds" echo "Started first node, sleeping for 60 seconds"
sleep 60 sleep 60
echo "Adding additional nodes" echo "Adding additional nodes"
ssh $NODE02 "sudo screen -dmSL k3s k3s server --server https://$3:6443 --token=test" ssh $NODE02 "sudo screen -dmSL k3s k3s server --server https://$kube_vip_vip:6443 --token=test"
ssh $NODE03 "sudo screen -dmSL k3s k3s server --server https://$3:6443 --token=test" ssh $NODE03 "sudo screen -dmSL k3s k3s server --server https://$kube_vip_vip:6443 --token=test"
sleep 20 sleep 20
ssh $NODE01 "sudo k3s kubectl get node -o wide" ssh $NODE01 "sudo k3s kubectl get node -o wide"

144
testing/kubeadm/create.sh
View File

@@ -1,4 +1,67 @@
#!/bin/bash #!/bin/bash
set -e
# Read node configuration
source ./testing/nodes
# Read logging function
source ./testing/logging.bash
install_deps() {
echo "Installing Kubernetes dependencies for Kubernetes $kubernetes_version on all nodes"
ssh $NODE01 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$kubernetes_version-00 kubectl=$kubernetes_version-00 kubeadm=$kubernetes_version-00"
ssh $NODE02 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$kubernetes_version-00 kubectl=$kubernetes_version-00 kubeadm=$kubernetes_version-00"
ssh $NODE03 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$kubernetes_version-00 kubectl=$kubernetes_version-00 kubeadm=$kubernetes_version-00"
ssh $NODE04 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$kubernetes_version-00 kubectl=$kubernetes_version-00 kubeadm=$kubernetes_version-00"
ssh $NODE05 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$kubernetes_version-00 kubectl=$kubernetes_version-00 kubeadm=$kubernetes_version-00"
}
first_node() {
logr "INFO" "Creating First node!"
#ssh $NODE01 "sudo modprobe ip_vs_rr"
#ssh $NODE01 "sudo modprobe nf_conntrack"
logr "INFO" "$(ssh $NODE01 "docker rmi plndr/kube-vip:$kube_vip_version" 2>&1)"
# echo "echo "ip_vs | tee -a /etc/modules"
logr "INFO" "Creating Kube-vip.io Manifest"
ssh $NODE01 "sudo docker run --network host --rm plndr/kube-vip:$kube_vip_version manifest pod --interface ens160 --vip $kube_vip_vip --arp --leaderElection --enableLoadBalancer $kube_vip_mode | sed \"s/image:.*/image: plndr\/kube-vip:$kube_vip_version/\" | sudo tee /etc/kubernetes/manifests/vip.yaml" >> $logfile
logr "INFO" "Deploying first Kubernetes node $NODE01"
FIRST_NODE=$(ssh $NODE01 "sudo kubeadm init --kubernetes-version $kubernetes_version --control-plane-endpoint $kube_vip_vip --upload-certs --pod-network-cidr=10.0.0.0/16")
echo "$FIRST_NODE" >> $logfile
CONTROLPLANE_CMD=$(echo "$FIRST_NODE" | grep -m1 certificate-key)
#CONTROLPLANE_CMD=$(ssh $NODE01 "sudo kubeadm init --kubernetes-version $kubernetes_version --control-plane-endpoint $kube_vip_vip --upload-certs --pod-network-cidr=10.0.0.0/16 | grep -m1 certificate-key")
ssh $NODE01 "sudo rm -rf ~/.kube/"
ssh $NODE01 "mkdir -p .kube"
ssh $NODE01 "sudo cp -i /etc/kubernetes/admin.conf .kube/config"
ssh $NODE01 "sudo chown dan:dan .kube/config"
logr "INFO" "Enabling strict ARP on kube-proxy"
ssh $NODE01 "kubectl get configmap kube-proxy -n kube-system -o yaml | sed -e \"s/strictARP: false/strictARP: true/\" | kubectl apply -f - -n kube-system"
ssh $NODE01 "kubectl describe configmap -n kube-system kube-proxy | grep strictARP"
logr "INFO" "Deploying Calico to the Kubernetes Cluster"
ssh $NODE01 "kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml" >> $logfile
logr "INFO" "Retrieving Join command"
JOIN_CMD=$(ssh $NODE01 " sudo kubeadm token create --print-join-command 2> /dev/null")
}
additional_controlplane() {
logr "INFO" "Adding $NODE02"
ssh $NODE02 "sudo $JOIN_CMD $CONTROLPLANE_CMD" >> $logfile
sleep 1
ssh $NODE02 "sudo docker run --network host --rm plndr/kube-vip:$kube_vip_version manifest pod --interface ens160 --vip $kube_vip_vip --arp --leaderElection --enableLoadBalancer $kube_vip_mode | sed \"s/image:.*/image: plndr\/kube-vip:$kube_vip_version/\"| sudo tee /etc/kubernetes/manifests/vip.yaml" >> $logfile
logr "INFO" "Adding $NODE03"
ssh $NODE03 "sudo $JOIN_CMD $CONTROLPLANE_CMD" >> $logfile
sleep 1
ssh $NODE03 "sudo docker run --network host --rm plndr/kube-vip:$kube_vip_version manifest pod --interface ens160 --vip $kube_vip_vip --arp --leaderElection --enableLoadBalancer $kube_vip_mode | sed \"s/image:.*/image: plndr\/kube-vip:$kube_vip_version/\"| sudo tee /etc/kubernetes/manifests/vip.yaml" >> $logfile
}
## Main()
# Ensure we have an entirely new logfile
reset_logfile
logr "INFO" "Starting kube-vip.io testing with Kubeadm"
logr "DEFAULT" "Creating Logfile $logfile"
if [[ -z $1 && -z $2 && -z $3 && -z $4 ]]; then if [[ -z $1 && -z $2 && -z $3 && -z $4 ]]; then
echo "Usage:" echo "Usage:"
@@ -7,77 +70,40 @@ if [[ -z $1 && -z $2 && -z $3 && -z $4 ]]; then
echo " Param 3: Kube-Vip mode [\"controlplane\"/\"services\"/\"hybrid\"]" echo " Param 3: Kube-Vip mode [\"controlplane\"/\"services\"/\"hybrid\"]"
echo " Param 4: Vip address" echo " Param 4: Vip address"
echo "" echo ""
echo "" ./create_k8s.sh 1.18.5 0.4.0 hybrid 192.168.0.40 echo "" ./create.sh 1.18.5 0.4.0 hybrid 192.168.0.40
exit 1 exit 1
fi fi
case "$3" in # Sane variable renaming
kubernetes_version=$1
kube_vip_version=$2
kube_vip_vip=$4
"controlplane") echo "Creating in control plane only mode" case "$3" in
mode="--controlplane" "controlplane") logr "INFO" "Creating in control plane only mode"
kube_vip_mode="--controlplane"
;; ;;
"services") echo "Creating in services-only mode" "services") logr "INFO" "Creating in services-only mode"
mode="--services" kube_vip_mode="--services"
;; ;;
"hybrid") echo "Creating in hybrid mode" "hybrid") logr "INFO" "Creating in hybrid mode"
mode="--controlplane --services" kube_vip_mode="--controlplane --services"
;; ;;
*) echo "Unknown kube-vip mode [$3]" *) echo "Unknown kube-vip mode [$3]"
exit -1 exit -1
;; ;;
esac esac
install_deps() { first_node
echo "Installing Kubernetes dependencies for Kubernetes $! on all nodes" additional_controlplane
ssh $NODE01 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$1-00 kubectl=$1-00 kubeadm=$1-00" logr "INFO" "Adding $NODE04"
ssh $NODE02 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$1-00 kubectl=$1-00 kubeadm=$1-00" ssh $NODE04 "sudo $JOIN_CMD" >> $logfile
ssh $NODE03 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$1-00 kubectl=$1-00 kubeadm=$1-00" logr "INFO" "Adding $NODE05"
ssh $NODE04 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$1-00 kubectl=$1-00 kubeadm=$1-00" ssh $NODE05 "sudo $JOIN_CMD" >> $logfile
ssh $NODE05 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add && sudo apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\" && sudo apt-get update -q && sudo apt-get install -qy kubelet=$1-00 kubectl=$1-00 kubeadm=$1-00" logr "DEFAULT" "Nodes should be deployed at this point, waiting 5 secs and querying the deployment"
}
source ./testing/nodes
first_node() {
echo "Creating First node!"
#ssh $NODE01 "sudo modprobe ip_vs_rr"
#ssh $NODE01 "sudo modprobe nf_conntrack"
ssh $NODE01 "docker rmi plndr/kube-vip:dev"
# echo "echo "ip_vs | tee -a /etc/modules"
ssh $NODE01 "sudo docker run --network host --rm plndr/kube-vip:$2 manifest pod --interface ens160 --vip $4 --arp --leaderElection --enableLoadBalancer $5 | sed \"s/image:.*/image: plndr\/kube-vip:dev/\" | sudo tee /etc/kubernetes/manifests/vip.yaml"
CONTROLPLANE_CMD=$(ssh $NODE01 "sudo kubeadm init --kubernetes-version $1 --control-plane-endpoint $4 --upload-certs --pod-network-cidr=10.0.0.0/16 | grep certificate-key")
ssh $NODE01 "sudo rm -rf ~/.kube/"
ssh $NODE01 "mkdir -p .kube"
ssh $NODE01 "sudo cp -i /etc/kubernetes/admin.conf .kube/config"
ssh $NODE01 "sudo chown dan:dan .kube/config"
echo "Enabling strict ARP on kube-proxy"
ssh $NODE01 "kubectl get configmap kube-proxy -n kube-system -o yaml | sed -e \"s/strictARP: false/strictARP: true/\" | kubectl apply -f - -n kube-system"
ssh $NODE01 "kubectl describe configmap -n kube-system kube-proxy | grep strictARP"
ssh $NODE01 "kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml"
JOIN_CMD=$(ssh $NODE01 " sudo kubeadm token create --print-join-command 2> /dev/null")
}
additional_controlplane() {
ssh $NODE02 "sudo $JOIN_CMD $CONTROLPLANE_CMD"
sleep 3
ssh $NODE02 "sudo docker run --network host --rm plndr/kube-vip:$1 manifest pod --interface ens160 --vip $2 --arp --leaderElection --enableLoadBalancer $3 | sed \"s/image:.*/image: plndr\/kube-vip:dev/\"| sudo tee /etc/kubernetes/manifests/vip.yaml"
ssh $NODE03 "sudo $JOIN_CMD $CONTROLPLANE_CMD"
sleep 3
ssh $NODE03 "sudo docker run --network host --rm plndr/kube-vip:$1 manifest pod --interface ens160 --vip $2 --arp --leaderElection --enableLoadBalancer $3 | sed \"s/image:.*/image: plndr\/kube-vip:dev/\"| sudo tee /etc/kubernetes/manifests/vip.yaml"
}
first_node $1 $2 $3 $4 $mode
additional_controlplane $2 $4 $mode
ssh $NODE04 "sudo $JOIN_CMD"
ssh $NODE05 "sudo $JOIN_CMD"
echo echo
echo " Nodes should be deployed at this point, waiting 5 secs and querying the deployment"
sleep 5 sleep 5
ssh $NODE01 "kubectl get nodes" ssh $NODE01 "kubectl get nodes" | tee >> $logfile
ssh $NODE01 "kubectl get pods -A" ssh $NODE01 "kubectl get pods -A" | tee >> $logfile
echo echo
echo "Kubernetes: $1, Kube-vip $2, Advertising VIP: $4" logr "INFO" "Kubernetes: $kubernetes_version, Kube-vip $kube_vip_version, Advertising VIP: $kube_vip_vip"

34
testing/kubeadm/service.sh Executable file
View File

@@ -0,0 +1,34 @@
#!/bin/bash
set -e
# Read node configuration
source ./testing/nodes
# Read logging function
source ./testing/logging.bash
logr "INFO" "Starting kube-vip.io service testing with Kubeadm"
logr "DEFAULT" "Creating Logfile $logfile"
# Adding Controller
logr "INFO" "Creating network range configmap"
ssh $NODE01 "kubectl create configmap -n kube-system kubevip --from-literal range-global=192.168.0.220-192.168.0.222" >> $logfile
logr "INFO" "Deploying kube-vip.io Controller"
ssh $NODE01 "kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml" >> $logfile
logr "INFO" "Creating \"nginx\" deployment"
ssh $NODE01 "kubectl apply -f https://k8s.io/examples/application/deployment.yaml" >> $logfile
sleep 5
logr "DEFAULT" "Creating \"nginx\" service"
ssh $NODE01 "kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx" >> $logfile
logr "INFO" "Sleeping for 20 seconds to give the controller time to \"reconcile\""
sleep 20
logr "INFO" "Retrieving logs from kube-vip.io cloud provider"
ssh $NODE01 "kubectl logs -n kube-system kube-vip-cloud-provider-0" >> $logfile
logr "INFO" "Retrieving service configuration"
ssh $NODE01 "kubectl describe svc nginx" | tee >> $logfile

46
testing/logging.bash Normal file
View File

@@ -0,0 +1,46 @@
LOG_ON_FILE=true
logfile="/tmp/kube-vip-testing.$(date +'%Y-%m-%d').log"
echo_timing() {
#--------------------------------------------------------
# Out: [19/01/2020 18h19:56] Hello
#--------------------------------------------------------
echo [`date +%d"/"%m"/"%Y" "%H"h"%M":"%S`] $@
}
echo_color(){
COLOR=$1; MSG=$2;
if [[ ${COLOR} == *"WHITE"* ]]; then echo -e "\\e[39m"${MSG}"\\e[0m";
elif [[ ${COLOR} == *"RED"* ]]; then echo -e "\\e[31m"${MSG}"\\e[0m";
elif [[ ${COLOR} == *"GREEN"* ]]; then echo -e "\\e[32m"${MSG}"\\e[0m";
elif [[ ${COLOR} == *"YELLOW"* ]]; then echo -e "\\e[33m"${MSG}"\\e[0m";
elif [[ ${COLOR} == *"BLUE"* ]]; then echo -e "\\e[34m"${MSG}"\\e[0m";
fi;
}
echo_console(){
TYPE_OF_MSG=$1; MSG=$2;
if [[ ${TYPE_OF_MSG} == *"1"* ]] || [[ ${TYPE_OF_MSG} == *"SUCCESS"* ]]; then echo_timing "$(echo_color "GREEN" "[+]: ${MSG}")";
elif [[ ${TYPE_OF_MSG} == *"2"* ]] || [[ ${TYPE_OF_MSG} == *"FAIL"* ]]; then echo_timing "$(echo_color "RED" "[-]: ${MSG}")";
elif [[ ${TYPE_OF_MSG} == *"3"* ]] || [[ ${TYPE_OF_MSG} == *"WARNING"* ]]; then echo_timing "$(echo_color "YELLOW" "[!]: ${MSG}")";
elif [[ ${TYPE_OF_MSG} == *"4"* ]] || [[ ${TYPE_OF_MSG} == *"INFO"* ]]; then echo_timing "$(echo_color "BLUE" "[i]: ${MSG}")";
elif [[ ${TYPE_OF_MSG} == *"0"* ]] || [[ ${TYPE_OF_MSG} == *"DEFAULT"* ]]; then echo_timing "$(echo_color "WHITE" "[:]: ${MSG}")";
else MSG=${TYPE_OF_MSG}; echo_timing "$(echo_color "WHITE" "[:]: ${MSG}")";
fi;
}
logr(){
TYPE_OF_MSG=$1; MSG=$2;
if [[ ${LOG_ON_FILE} ]]; then echo_console "${TYPE_OF_MSG}" "${MSG}" | tee -a "${logfile}";
else echo_console "${TYPE_OF_MSG}" "${MSG}"; fi;
}
reset_logfile() {
touch $logfile
cat /dev/null > $logfile
}