add Kubernetes End to End tests to the NFS Driver

Gopkg.toml

@@ -42,22 +42,22 @@
 
 [[constraint]]
   name = "k8s.io/kubernetes"
-  version = "v1.12.0"
+  version = "v1.14.0"
 
 [[override]]
   name = "k8s.io/kubernetes"
-  version = "v1.12.0"
+  version = "v1.14.0"
 
 [[override]]
-  version = "kubernetes-1.12.0"
+  version = "kubernetes-1.14.0"
   name = "k8s.io/api"
 
 [[override]]
-  version = "kubernetes-1.12.0"
+  version = "kubernetes-1.14.0"
   name = "k8s.io/apiserver"
 
 [[override]]
-  version = "kubernetes-1.12.0"
+  version = "kubernetes-1.14.0"
   name = "k8s.io/apiextensions-apiserver"
 
 [[override]]
@@ -68,10 +68,18 @@
   name = "github.com/json-iterator/go"
   version = "1.1.4"
 
+[[override]]
+  name = "k8s.io/client-go"
+  branch = "master"
+
 [[override]]
   name = "gopkg.in/square/go-jose.v2"
   version = "2.1.7"
 
+[[override]]
+  source = "https://github.com/fsnotify/fsnotify/archive/v1.4.7.tar.gz"
+  name = "gopkg.in/fsnotify.v1"
+
 [prune]
   non-go = true
   go-tests = true

README.md

@@ -65,6 +65,27 @@ nfstestvol
 $ csc node get-id --endpoint tcp://127.0.0.1:10000
 CSINode
 ```
+## Running Kubernetes End To End tests on an NFS Driver
+
+First, stand up a local cluster `ALLOW_PRIVILEGED=1 hack/local-up-cluster.sh` (from your Kubernetes repo)
+For Fedora/RHEL clusters, the following might be required:
+  ```
+  sudo chown -R $USER:$USER /var/run/kubernetes/
+  sudo chown -R $USER:$USER /var/lib/kubelet
+  sudo chcon -R -t svirt_sandbox_file_t /var/lib/kubelet
+  ```
+If you are plannig to test using your own private image, you could either install your nfs driver using your own set of YAML files, or edit the existing YAML files to use that private image.
+
+When using the [existing set of YAML files](https://github.com/kubernetes-csi/csi-driver-nfs/tree/master/deploy/kubernetes), you would edit the [csi-attacher-nfsplugin.yaml](https://github.com/kubernetes-csi/csi-driver-nfs/blob/master/deploy/kubernetes/csi-attacher-nfsplugin.yaml#L46) and [csi-nodeplugin-nfsplugin.yaml](https://github.com/kubernetes-csi/csi-driver-nfs/blob/master/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml#L45) files to include your private image instead of the default one. After editing these files, skip to step 3 of the following steps.
+
+If you already have a driver installed, skip to step 4 of the following steps.
+
+1) Build the nfs driver by running `make`
+2) Create NFS Driver Image, where the image tag would be whatever that is required by your YAML deployment files        `docker build -t quay.io/k8scsi/nfsplugin:v1.0.0 .`
+3) Install the Driver: `kubectl create -f deploy/kubernetes`
+4) Build E2E test binary: `make build-tests`
+5) Run E2E Tests using the following command: `./bin/tests --ginkgo.v --ginkgo.progress --kubeconfig=/var/run/kubernetes/admin.kubeconfig`
+
 
 ## Community, discussion, contribution, and support
 

cmd/tests/nfs-e2e.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	_ "github.com/kubernetes-csi/csi-driver-nfs/test"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"testing"
+)
+
+func init() {
+	framework.HandleFlags()
+	framework.AfterReadingAllFlags(&framework.TestContext)
+}
+
+func Test(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "CSI Suite")
+}
+
+func main() {
+	Test(&testing.T{})
+}

pkg/nfs/controllerserver.go

@@ -0,0 +1,23 @@
+package nfs
+
+import (
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/kubernetes-csi/drivers/pkg/csi-common"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type ControllerServer struct {
+	*csicommon.DefaultControllerServer
+}
+
+func (cs ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+func getControllerServer(csiDriver *csicommon.CSIDriver) ControllerServer {
+	return ControllerServer{
+		csicommon.NewDefaultControllerServer(csiDriver),
+	}
+}

pkg/nfs/driver.go

@@ -72,7 +72,7 @@ func (d *driver) Run() {
 		csicommon.NewDefaultIdentityServer(d.csiDriver),
 		// NFS plugin has not implemented ControllerServer
 		// using default controllerserver.
-		csicommon.NewDefaultControllerServer(d.csiDriver),
+		getControllerServer(d.csiDriver),
 		NewNodeServer(d))
 	s.Wait()
 }

pkg/nfs/nodeserver.go

@@ -22,13 +22,11 @@ import (
 	"strings"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/kubernetes-csi/drivers/pkg/csi-common"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"k8s.io/kubernetes/pkg/util/mount"
-	"k8s.io/kubernetes/pkg/volume/util"
-
-	"github.com/kubernetes-csi/drivers/pkg/csi-common"
 )
 
 type nodeServer struct {
@@ -92,7 +90,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 		return nil, status.Error(codes.NotFound, "Volume not mounted")
 	}
 
-	err = util.UnmountPath(req.GetTargetPath(), mount.New(""))
+	err = mount.CleanupMountPoint(req.GetTargetPath(), mount.New(""), false)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
@@ -107,3 +105,7 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
 func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
 	return &csi.NodeStageVolumeResponse{}, nil
 }
+
+func (ns *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "")
+}

test/csi-volumes.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package test
+
+import (
+	. "github.com/onsi/ginkgo"
+	_ "github.com/onsi/gomega"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
+	"path"
+)
+
+var CSITestSuites = []func() testsuites.TestSuite{
+	testsuites.InitVolumesTestSuite,
+	testsuites.InitVolumeIOTestSuite,
+	testsuites.InitVolumeModeTestSuite,
+	testsuites.InitSubPathTestSuite,
+	testsuites.InitProvisioningTestSuite,
+	//testsuites.InitSnapshottableTestSuite,
+	//testsuites.InitMultiVolumeTestSuite,
+}
+
+// This executes testSuites for csi volumes.
+var _ = utils.SIGDescribe("CSI Volumes", func() {
+	testfiles.AddFileSource(testfiles.RootFileSource{Root: path.Join(framework.TestContext.RepoRoot, "../../deploy/kubernetes/")})
+
+	curDriver := NFSdriver()
+	Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() {
+		testsuites.DefineTestSuite(curDriver, CSITestSuites)
+	})
+
+})

test/nfs-testdriver.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package test
+
+import (
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+)
+
+type nfsDriver struct {
+	driverInfo testsuites.DriverInfo
+	manifests  []string
+}
+
+var NFSdriver = InitNFSDriver
+
+type nfsVolume struct {
+	serverIP  string
+	serverPod *v1.Pod
+	f         *framework.Framework
+}
+
+// initNFSDriver returns nfsDriver that implements TestDriver interface
+func initNFSDriver(name string, manifests ...string) testsuites.TestDriver {
+	return &nfsDriver{
+		driverInfo: testsuites.DriverInfo{
+			Name:        name,
+			MaxFileSize: testpatterns.FileSizeLarge,
+			SupportedFsType: sets.NewString(
+				"", // Default fsType
+			),
+			Capabilities: map[testsuites.Capability]bool{
+				testsuites.CapPersistence: true,
+				testsuites.CapExec:        true,
+			},
+		},
+		manifests: manifests,
+	}
+}
+
+func InitNFSDriver() testsuites.TestDriver {
+
+	return initNFSDriver("csi-nfsplugin",
+		"csi-attacher-nfsplugin.yaml",
+		"csi-attacher-rbac.yaml",
+		"csi-nodeplugin-nfsplugin.yaml",
+		"csi-nodeplugin-rbac.yaml")
+
+}
+
+var _ testsuites.TestDriver = &nfsDriver{}
+var _ testsuites.PreprovisionedVolumeTestDriver = &nfsDriver{}
+var _ testsuites.PreprovisionedPVTestDriver = &nfsDriver{}
+
+func (n *nfsDriver) GetDriverInfo() *testsuites.DriverInfo {
+	return &n.driverInfo
+}
+
+func (n *nfsDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+	if pattern.VolType == testpatterns.DynamicPV {
+		framework.Skipf("NFS Driver does not support dynamic provisioning -- skipping")
+	}
+}
+
+func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, volume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+	nv, _ := volume.(*nfsVolume)
+	return &v1.PersistentVolumeSource{
+		CSI: &v1.CSIPersistentVolumeSource{
+			Driver:       n.driverInfo.Name,
+			VolumeHandle: "nfs-vol",
+			VolumeAttributes: map[string]string{
+				"server":   nv.serverIP,
+				"share":    "/",
+				"readOnly": "true",
+			},
+		},
+	}, nil
+}
+
+func (n *nfsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+	config := &testsuites.PerTestConfig{
+		Driver:    n,
+		Prefix:    "nfs",
+		Framework: f,
+	}
+
+	return config, func() {}
+}
+
+func (n *nfsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+	f := config.Framework
+	cs := f.ClientSet
+	ns := f.Namespace
+
+	switch volType {
+	case testpatterns.InlineVolume:
+		fallthrough
+	case testpatterns.PreprovisionedPV:
+
+		//Create nfs server pod
+		c := framework.VolumeTestConfig{
+			Namespace:          ns.Name,
+			Prefix:             "nfs",
+			ServerImage:        "gcr.io/kubernetes-e2e-test-images/volume/nfs:1.0",
+			ServerPorts:        []int{2049},
+			ServerVolumes:      map[string]string{"": "/exports"},
+			ServerReadyMessage: "NFS started",
+		}
+		config.ServerConfig = &c
+		serverPod, serverIP := framework.CreateStorageServer(cs, c)
+
+		return &nfsVolume{
+			serverIP:  serverIP,
+			serverPod: serverPod,
+			f:         f,
+		}
+
+	case testpatterns.DynamicPV:
+		// Do nothing
+	default:
+		framework.Failf("Unsupported volType:%v is specified", volType)
+	}
+	return nil
+}
+
+func (v *nfsVolume) DeleteVolume() {
+	framework.CleanUpVolumeServer(v.f, v.serverPod)
+}