From 0ad6d94899d00835b1f65327a6e06d648eec75fd Mon Sep 17 00:00:00 2001 From: Mayank Shah Date: Tue, 20 Oct 2020 03:54:23 +0530 Subject: [PATCH] CI: Add K8s external storage E2E tests Signed-off-by: Mayank Shah --- .github/workflows/e2e.yaml | 39 +++++++ Makefile | 27 +++++ .../manifests/csi-nfs-controller.yaml | 108 ++++++++++++++++++ test/e2e_external/manifests/csi-nfs-node.yaml | 77 +++++++++++++ test/e2e_external/run.sh | 21 ++++ test/e2e_external/testdriver.yaml | 10 ++ 6 files changed, 282 insertions(+) create mode 100644 .github/workflows/e2e.yaml create mode 100644 test/e2e_external/manifests/csi-nfs-controller.yaml create mode 100644 test/e2e_external/manifests/csi-nfs-node.yaml create mode 100644 test/e2e_external/run.sh create mode 100644 test/e2e_external/testdriver.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..466afe2c2 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,39 @@ +name: K8s external storage E2E Tests +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] +jobs: + test: + name: E2E Test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + id: go + - name: Build image + run: | + make build + docker build -t nfsplugin:latest . + - uses: engineerd/setup-kind@v0.4.0 + name: Installing KinD cluster + with: + version: "v0.9.0" + - name: Push image to KinD + run: | + kind load docker-image --name=kind nfsplugin:latest + - name: Install NFS Server on KinD + run: kubectl apply -f ./examples/kubernetes/nfs-server/nfs-server.yaml + - name: Install csi-nfs driver on KinD + run: make kind-install-driver + - name: Setup K8s E2E external storage tests + run: make setup-external-e2e + - name: Run tests + run: | + go get -u github.com/onsi/ginkgo/ginkgo + make run-external-e2e \ No newline at end of file diff --git a/Makefile b/Makefile index 4354ea045..5d715cf25 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ CMDS=nfsplugin DEPLOY_FOLDER = ./deploy/kubernetes CMDS=nfsplugin +LOCAL_REGISTRY=localhost:5000 all: build include release-tools/build.make @@ -44,3 +45,29 @@ local-k8s-uninstall: kubectl delete -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml --ignore-not-found kubectl delete -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml --ignore-not-found echo "Uninstalled NFS driver" + + +.PHONY: kind-install-driver +kind-install-driver: + echo "Instlling locally" + kubectl apply -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml + kubectl apply -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml + kubectl apply -f ./test/e2e_external/manifests/csi-nfs-controller.yaml + kubectl apply -f ./test/e2e_external/manifests/csi-nfs-node.yaml + echo "Successfully installed" + +.PHONY: setup-external-e2e +setup-external-e2e: + curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.19.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz + tar -xvf e2e-tests.tar.gz + rm e2e-tests.tar.gz + mkdir /tmp/csi-nfs + cp ./kubernetes/test/bin/e2e.test /tmp/csi-nfs/e2e.test + rm -r kubernetes + cp ./examples/kubernetes/storageclass-nfs.yaml /tmp/csi-nfs/storageclass.yaml + cp ./test/e2e_external/testdriver.yaml /tmp/csi-nfs/testdriver.yaml + +.PHONY: run-external-e2e +run-external-e2e: + bash ./test/e2e_external/run.sh + diff --git a/test/e2e_external/manifests/csi-nfs-controller.yaml b/test/e2e_external/manifests/csi-nfs-controller.yaml new file mode 100644 index 000000000..945e1b762 --- /dev/null +++ b/test/e2e_external/manifests/csi-nfs-controller.yaml @@ -0,0 +1,108 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-nfs-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: csi-nfs-controller + template: + metadata: + labels: + app: csi-nfs-controller + spec: + serviceAccountName: csi-nfs-controller-sa + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "true" + effect: "NoSchedule" + containers: + - name: csi-provisioner + image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v1.4.0 + args: + - "-v=5" + - "--csi-address=$(ADDRESS)" + - "--enable-leader-election" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + - --health-port=29642 + - --v=5 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: nfs + image: nfsplugin:latest + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /csi + name: socket-dir + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: socket-dir + emptyDir: {} diff --git a/test/e2e_external/manifests/csi-nfs-node.yaml b/test/e2e_external/manifests/csi-nfs-node.yaml new file mode 100644 index 000000000..fc8156190 --- /dev/null +++ b/test/e2e_external/manifests/csi-nfs-node.yaml @@ -0,0 +1,77 @@ +# This YAML file contains driver-registrar & csi driver nodeplugin API objects +# that are necessary to run CSI nodeplugin for nfs +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-nfs-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-nfs-node + template: + metadata: + labels: + app: csi-nfs-node + spec: + hostNetwork: true # original nfs connection would be broken without hostNetwork setting + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"] + args: + - --v=5 + - --csi-address=/plugin/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: registration-dir + mountPath: /registration + - name: nfs + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: nfsplugin:latest + args : + - "-v=5" + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://plugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir diff --git a/test/e2e_external/run.sh b/test/e2e_external/run.sh new file mode 100644 index 000000000..de3d7a38b --- /dev/null +++ b/test/e2e_external/run.sh @@ -0,0 +1,21 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash +ginkgo -p --progress --v -focus='External.Storage.*csi-nfsplugin' \ + -skip='\[Disruptive\]' \ + /tmp/csi-nfs/e2e.test \ + -- \ + -storage.testdriver=/tmp/csi-nfs/testdriver.yaml \ + --kubeconfig=$HOME/.kube/config \ No newline at end of file diff --git a/test/e2e_external/testdriver.yaml b/test/e2e_external/testdriver.yaml new file mode 100644 index 000000000..658cb01f7 --- /dev/null +++ b/test/e2e_external/testdriver.yaml @@ -0,0 +1,10 @@ +StorageClass: + FromName: false + FromFile: /tmp/csi-nfs/storageclass.yaml +DriverInfo: + Name: csi-nfsplugin + Capabilities: + persistence: true + exec: true + multipods: true + RWX: true \ No newline at end of file