Skip to content

Commit 3aeb3d0

Browse files
mayankshah1607mayankshah1607
committed
CI: Add K8s external storage E2E tests
Signed-off-by: Mayank Shah <[email protected]>
1 parent 108aef3 commit 3aeb3d0

File tree

6 files changed

+280
-0
lines changed

6 files changed

+280
-0
lines changed

‎.github/workflows/e2e.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: K8s external storage E2E Tests
2+
on:
3+
push:
4+
branches: [ master ]
5+
pull_request:
6+
branches: [ master ]
7+
jobs:
8+
test:
9+
name: E2E Test
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout code
13+
uses: actions/checkout@v2
14+
- name: Set up Go 1.x
15+
uses: actions/setup-go@v2
16+
with:
17+
go-version: ^1.13
18+
id: go
19+
- name: Build image
20+
run: |
21+
make build
22+
docker build -t nfsplugin:latest .
23+
- uses: engineerd/[email protected]
24+
name: Installing KinD cluster
25+
with:
26+
version: "v0.9.0"
27+
- name: Push image to KinD
28+
run: |
29+
kind load docker-image --name=kind nfsplugin:latest
30+
- name: Install NFS Server on KinD
31+
run: kubectl apply -f ./examples/kubernetes/nfs-server/nfs-server.yaml
32+
- name: Install csi-nfs driver on KinD
33+
run: make kind-install-driver
34+
- name: Setup K8s E2E external storage tests
35+
run: make setup-external-e2e
36+
- name: Run tests
37+
run: |
38+
go get -u github.com/onsi/ginkgo/ginkgo
39+
make run-external-e2e

‎Makefile

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
CMDS=nfsplugin
1515
DEPLOY_FOLDER = ./deploy/kubernetes
1616
CMDS=nfsplugin
17+
LOCAL_REGISTRY=localhost:5000
1718
all: build
1819

1920
include release-tools/build.make
@@ -44,3 +45,29 @@ local-k8s-uninstall:
4445
kubectl delete -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml --ignore-not-found
4546
kubectl delete -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml --ignore-not-found
4647
echo "Uninstalled NFS driver"
48+
49+
50+
.PHONY: kind-install-driver
51+
kind-install-driver:
52+
echo "Instlling locally"
53+
kubectl apply -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml
54+
kubectl apply -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml
55+
kubectl apply -f ./test/e2e/manifests/csi-nfs-controller.yaml
56+
kubectl apply -f ./test/e2e/manifests/csi-nfs-node.yaml
57+
echo "Successfully installed"
58+
59+
.PHONY: setup-external-e2e
60+
setup-external-e2e:
61+
curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.14.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz
62+
tar -xvf e2e-tests.tar.gz
63+
rm e2e-tests.tar.gz
64+
mkdir /tmp/csi-nfs
65+
cp ./kubernetes/test/bin/e2e.test /tmp/csi-nfs/e2e.test
66+
rm -r kubernetes
67+
cp ./examples/kubernetes/storageclass-nfs.yaml /tmp/csi-nfs/storageclass.yaml
68+
cp ./test/e2e/testdriver.yaml /tmp/csi-nfs/testdriver.yaml
69+
70+
.PHONY: run-external-e2e
71+
run-external-e2e:
72+
bash ./test/e2e/run.sh
73+

‎test/e2e/manifests/csi-nfs-controller.yaml

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
---
2+
kind: Deployment
3+
apiVersion: apps/v1
4+
metadata:
5+
name: csi-nfs-controller
6+
namespace: kube-system
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: csi-nfs-controller
12+
template:
13+
metadata:
14+
labels:
15+
app: csi-nfs-controller
16+
spec:
17+
serviceAccountName: csi-nfs-controller-sa
18+
nodeSelector:
19+
kubernetes.io/os: linux
20+
priorityClassName: system-cluster-critical
21+
tolerations:
22+
- key: "node-role.kubernetes.io/master"
23+
operator: "Equal"
24+
value: "true"
25+
effect: "NoSchedule"
26+
containers:
27+
- name: csi-provisioner
28+
image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v1.4.0
29+
args:
30+
- "-v=5"
31+
- "--csi-address=$(ADDRESS)"
32+
- "--enable-leader-election"
33+
- "--leader-election-type=leases"
34+
env:
35+
- name: ADDRESS
36+
value: /csi/csi.sock
37+
volumeMounts:
38+
- mountPath: /csi
39+
name: socket-dir
40+
resources:
41+
limits:
42+
cpu: 100m
43+
memory: 100Mi
44+
requests:
45+
cpu: 10m
46+
memory: 20Mi
47+
- name: liveness-probe
48+
image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v1.1.0
49+
args:
50+
- --csi-address=/csi/csi.sock
51+
- --connection-timeout=3s
52+
- --health-port=29642
53+
- --v=5
54+
volumeMounts:
55+
- name: socket-dir
56+
mountPath: /csi
57+
resources:
58+
limits:
59+
cpu: 100m
60+
memory: 100Mi
61+
requests:
62+
cpu: 10m
63+
memory: 20Mi
64+
- name: nfs
65+
image: nfsplugin:latest
66+
securityContext:
67+
privileged: true
68+
capabilities:
69+
add: ["SYS_ADMIN"]
70+
allowPrivilegeEscalation: true
71+
imagePullPolicy: IfNotPresent
72+
args:
73+
- "-v=5"
74+
- "--nodeid=$(NODE_ID)"
75+
- "--endpoint=$(CSI_ENDPOINT)"
76+
env:
77+
- name: NODE_ID
78+
valueFrom:
79+
fieldRef:
80+
fieldPath: spec.nodeName
81+
- name: CSI_ENDPOINT
82+
value: unix:///csi/csi.sock
83+
volumeMounts:
84+
- name: plugin-dir
85+
mountPath: /plugin
86+
- name: pods-mount-dir
87+
mountPath: /var/lib/kubelet/pods
88+
mountPropagation: "Bidirectional"
89+
- mountPath: /csi
90+
name: socket-dir
91+
resources:
92+
limits:
93+
cpu: 200m
94+
memory: 200Mi
95+
requests:
96+
cpu: 10m
97+
memory: 20Mi
98+
volumes:
99+
- name: plugin-dir
100+
hostPath:
101+
path: /var/lib/kubelet/plugins/csi-nfsplugin
102+
type: DirectoryOrCreate
103+
- name: pods-mount-dir
104+
hostPath:
105+
path: /var/lib/kubelet/pods
106+
type: Directory
107+
- name: socket-dir
108+
emptyDir: {}

‎test/e2e/manifests/csi-nfs-node.yaml

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# This YAML file contains driver-registrar & csi driver nodeplugin API objects
2+
# that are necessary to run CSI nodeplugin for nfs
3+
kind: DaemonSet
4+
apiVersion: apps/v1
5+
metadata:
6+
name: csi-nfs-node
7+
namespace: kube-system
8+
spec:
9+
selector:
10+
matchLabels:
11+
app: csi-nfs-node
12+
template:
13+
metadata:
14+
labels:
15+
app: csi-nfs-node
16+
spec:
17+
hostNetwork: true # original nfs connection would be broken without hostNetwork setting
18+
dnsPolicy: ClusterFirstWithHostNet
19+
containers:
20+
- name: node-driver-registrar
21+
image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
22+
lifecycle:
23+
preStop:
24+
exec:
25+
command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"]
26+
args:
27+
- --v=5
28+
- --csi-address=/plugin/csi.sock
29+
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
30+
env:
31+
- name: KUBE_NODE_NAME
32+
valueFrom:
33+
fieldRef:
34+
fieldPath: spec.nodeName
35+
volumeMounts:
36+
- name: plugin-dir
37+
mountPath: /plugin
38+
- name: registration-dir
39+
mountPath: /registration
40+
- name: nfs
41+
securityContext:
42+
privileged: true
43+
capabilities:
44+
add: ["SYS_ADMIN"]
45+
allowPrivilegeEscalation: true
46+
image: nfsplugin:latest
47+
args :
48+
- "-v=5"
49+
- "--nodeid=$(NODE_ID)"
50+
- "--endpoint=$(CSI_ENDPOINT)"
51+
env:
52+
- name: NODE_ID
53+
valueFrom:
54+
fieldRef:
55+
fieldPath: spec.nodeName
56+
- name: CSI_ENDPOINT
57+
value: unix://plugin/csi.sock
58+
imagePullPolicy: "IfNotPresent"
59+
volumeMounts:
60+
- name: plugin-dir
61+
mountPath: /plugin
62+
- name: pods-mount-dir
63+
mountPath: /var/lib/kubelet/pods
64+
mountPropagation: "Bidirectional"
65+
volumes:
66+
- name: plugin-dir
67+
hostPath:
68+
path: /var/lib/kubelet/plugins/csi-nfsplugin
69+
type: DirectoryOrCreate
70+
- name: pods-mount-dir
71+
hostPath:
72+
path: /var/lib/kubelet/pods
73+
type: Directory
74+
- hostPath:
75+
path: /var/lib/kubelet/plugins_registry
76+
type: Directory
77+
name: registration-dir

‎test/e2e/run.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Copyright 2020 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
#!/bin/bash
16+
ginkgo -p --progress --v -focus='External.Storage.*csi-nfs' \
17+
-skip='\[Disruptive\]' \
18+
/tmp/csi-nfs/e2e.test \
19+
-- \
20+
-storage.testdriver=/tmp/csi-nfs/testdriver.yaml \
21+
--kubeconfig=$HOME/.kube/config

‎test/e2e/testdriver.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
StorageClass:
2+
FromName: false
3+
FromFile: /tmp/csi-nfs/storageclass.yaml
4+
DriverInfo:
5+
Name: nfs.csi.k8s.io
6+
Capabilities:
7+
persistence: true
8+
exec: true

0 commit comments

Comments
 (0)