From 2c9b076ea72c57d122f509d3c704de6ba30e626e Mon Sep 17 00:00:00 2001 From: Masaki Kimura Date: Thu, 6 Jun 2019 15:00:19 +0000 Subject: [PATCH 1/2] Add provision support as a simplenfs plugin --- .gitignore | 1 + Dockerfile | 2 + cmd/nfsplugin/main.go | 9 +- .../csi-attacher-nfsplugin.yaml | 79 +++++++ .../csi-attacher-rbac.yaml | 46 ++++ .../csi-nodeplugin-nfsplugin.yaml | 75 +++++++ .../csi-nodeplugin-rbac.yaml | 34 +++ examples/kubernetes-simplenfs/nginx.yaml | 19 ++ examples/kubernetes-simplenfs/pvc.yaml | 11 + .../kubernetes-simplenfs/storageclass.yaml | 9 + pkg/nfs/controllerserver.go | 50 +++++ pkg/nfs/nfs.go | 29 ++- release-tools/build.make | 6 +- simplenfs/plugin.go | 206 ++++++++++++++++++ 14 files changed, 561 insertions(+), 15 deletions(-) create mode 100644 deploy/kubernetes-simplenfs/csi-attacher-nfsplugin.yaml create mode 100644 deploy/kubernetes-simplenfs/csi-attacher-rbac.yaml create mode 100644 deploy/kubernetes-simplenfs/csi-nodeplugin-nfsplugin.yaml create mode 100644 deploy/kubernetes-simplenfs/csi-nodeplugin-rbac.yaml create mode 100644 examples/kubernetes-simplenfs/nginx.yaml create mode 100644 examples/kubernetes-simplenfs/pvc.yaml create mode 100644 examples/kubernetes-simplenfs/storageclass.yaml create mode 100644 simplenfs/plugin.go diff --git a/.gitignore b/.gitignore index 5e56e040e..ebe27f1ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /bin +simplenfs/bin diff --git a/Dockerfile b/Dockerfile index 13f5d2485..3ca6a54cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM centos:7.4.1708 # Copy nfsplugin from build _output directory COPY bin/nfsplugin /nfsplugin +RUN mkdir -p /simplenfs/bin +COPY simplenfs/bin/plugin.so /simplenfs/plugin.so RUN yum -y install nfs-utils && yum -y install epel-release && yum -y install jq && yum clean all diff --git a/cmd/nfsplugin/main.go b/cmd/nfsplugin/main.go index f757df0b0..ae144f338 100644 --- a/cmd/nfsplugin/main.go +++ b/cmd/nfsplugin/main.go @@ -27,8 +27,9 @@ import ( ) var ( - endpoint string - nodeID string + endpoint string + nodeID string + controllerPlugin string ) func init() { @@ -55,6 +56,8 @@ func main() { cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint") cmd.MarkPersistentFlagRequired("endpoint") + cmd.PersistentFlags().StringVar(&controllerPlugin, "controllerPlugin", "", "Controller plugin") + cmd.ParseFlags(os.Args[1:]) if err := cmd.Execute(); err != nil { fmt.Fprintf(os.Stderr, "%s", err.Error()) @@ -65,6 +68,6 @@ func main() { } func handle() { - d := nfs.NewNFSdriver(nodeID, endpoint) + d := nfs.NewNFSdriver(nodeID, endpoint, controllerPlugin) d.Run() } diff --git a/deploy/kubernetes-simplenfs/csi-attacher-nfsplugin.yaml b/deploy/kubernetes-simplenfs/csi-attacher-nfsplugin.yaml new file mode 100644 index 000000000..c53c35ab4 --- /dev/null +++ b/deploy/kubernetes-simplenfs/csi-attacher-nfsplugin.yaml @@ -0,0 +1,79 @@ +# This YAML file contains attacher & csi driver API objects that are necessary +# to run external CSI attacher for nfs + +kind: Service +apiVersion: v1 +metadata: + name: csi-attacher-nfsplugin + labels: + app: csi-attacher-nfsplugin +spec: + selector: + app: csi-attacher-nfsplugin + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: csi-attacher-nfsplugin +spec: + serviceName: "csi-attacher" + replicas: 1 + template: + metadata: + labels: + app: csi-attacher-nfsplugin + spec: + serviceAccount: csi-attacher + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.1 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: unix:///csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: nfs + image: nfsplugin:latest + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--controllerPlugin=/simplenfs/plugin.so" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://plugin/csi.sock + imagePullPolicy: "IfNotPresent" + securityContext: + capabilities: + add: + - SYS_ADMIN + volumeMounts: + - name: socket-dir + mountPath: /plugin + volumes: + - name: socket-dir + emptyDir: diff --git a/deploy/kubernetes-simplenfs/csi-attacher-rbac.yaml b/deploy/kubernetes-simplenfs/csi-attacher-rbac.yaml new file mode 100644 index 000000000..8073f1227 --- /dev/null +++ b/deploy/kubernetes-simplenfs/csi-attacher-rbac.yaml @@ -0,0 +1,46 @@ +# This YAML file contains RBAC API objects that are necessary to run external +# CSI attacher for nfs flex adapter + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-attacher + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-attacher-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-role +subjects: + - kind: ServiceAccount + name: csi-attacher + namespace: default +roleRef: + kind: ClusterRole + name: external-attacher-runner + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes-simplenfs/csi-nodeplugin-nfsplugin.yaml b/deploy/kubernetes-simplenfs/csi-nodeplugin-nfsplugin.yaml new file mode 100644 index 000000000..880cc73aa --- /dev/null +++ b/deploy/kubernetes-simplenfs/csi-nodeplugin-nfsplugin.yaml @@ -0,0 +1,75 @@ +# This YAML file contains driver-registrar & csi driver nodeplugin API objects +# that are necessary to run CSI nodeplugin for nfs +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: csi-nodeplugin-nfsplugin +spec: + selector: + matchLabels: + app: csi-nodeplugin-nfsplugin + template: + metadata: + labels: + app: csi-nodeplugin-nfsplugin + spec: + serviceAccount: csi-nodeplugin + hostNetwork: true + containers: + - name: node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"] + args: + - --v=5 + - --csi-address=/plugin/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: registration-dir + mountPath: /registration + - name: nfs + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: nfsplugin:latest + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://plugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir diff --git a/deploy/kubernetes-simplenfs/csi-nodeplugin-rbac.yaml b/deploy/kubernetes-simplenfs/csi-nodeplugin-rbac.yaml new file mode 100644 index 000000000..530e067b5 --- /dev/null +++ b/deploy/kubernetes-simplenfs/csi-nodeplugin-rbac.yaml @@ -0,0 +1,34 @@ +# This YAML defines all API objects to create RBAC roles for CSI node plugin +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nodeplugin + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin +subjects: + - kind: ServiceAccount + name: csi-nodeplugin + namespace: default +roleRef: + kind: ClusterRole + name: csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/examples/kubernetes-simplenfs/nginx.yaml b/examples/kubernetes-simplenfs/nginx.yaml new file mode 100644 index 000000000..2ce222581 --- /dev/null +++ b/examples/kubernetes-simplenfs/nginx.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-simplenfsplugin +spec: + containers: + - image: maersk/nginx + imagePullPolicy: Always + name: nginx + ports: + - containerPort: 80 + protocol: TCP + volumeMounts: + - mountPath: /var/www + name: data-simplenfsplugin + volumes: + - name: data-simplenfsplugin + persistentVolumeClaim: + claimName: data-simplenfsplugin diff --git a/examples/kubernetes-simplenfs/pvc.yaml b/examples/kubernetes-simplenfs/pvc.yaml new file mode 100644 index 000000000..244eab3bf --- /dev/null +++ b/examples/kubernetes-simplenfs/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-simplenfsplugin +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: csi-simplenfs-sc diff --git a/examples/kubernetes-simplenfs/storageclass.yaml b/examples/kubernetes-simplenfs/storageclass.yaml new file mode 100644 index 000000000..6a561d600 --- /dev/null +++ b/examples/kubernetes-simplenfs/storageclass.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-simplenfs-sc +provisioner: csi-nfsplugin +parameters: + server: 192.168.122.57 + rootpath: /exports +reclaimPolicy: Delete diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index 0bbbb4235..7c205e096 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -1,6 +1,8 @@ package nfs import ( + "plugin" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/glog" "golang.org/x/net/context" @@ -12,11 +14,51 @@ type ControllerServer struct { Driver *nfsDriver } +func isSupported(pluginName string, symbolName string) bool { + symbol, err := lookupSymbol(pluginName, symbolName) + return err == nil && symbol != nil +} + +func lookupSymbol(pluginName string, symbolName string) (interface{}, error) { + if pluginName != "" { + plug, err := plugin.Open(pluginName) + if err != nil { + glog.Infof("Failed to load plugin: %s error: %v", pluginName, err) + return nil, err + } + symbol, err := plug.Lookup(symbolName) + if err != nil { + glog.Infof("Failed to lookup symbol: %s error: %v", symbolName, err) + return nil, err + } + return symbol, nil + } + return nil, nil +} + func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + glog.Infof("CreateVolume called") + symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "CreateVolume") + if err == nil && symbol != nil { + createVolume, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error)) + if ok { + return createVolume(cs, ctx, req) + } + } + return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + glog.Infof("DeleteVolume called") + symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "DeleteVolume") + if err == nil && symbol != nil { + deleteVolume, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error)) + if ok { + return deleteVolume(cs, ctx, req) + } + } + return nil, status.Error(codes.Unimplemented, "") } @@ -29,6 +71,14 @@ func (cs *ControllerServer) ControllerUnpublishVolume(ctx context.Context, req * } func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "ValidateVolumeCapabilities") + if err == nil && symbol != nil { + validateVolumeCapabilities, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error)) + if ok { + return validateVolumeCapabilities(cs, ctx, req) + } + } + return nil, status.Error(codes.Unimplemented, "") } diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index c5378337c..c9609caaf 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -26,7 +26,8 @@ type nfsDriver struct { nodeID string version string - endpoint string + endpoint string + controllerPlugin string //ids *identityServer ns *nodeServer @@ -42,21 +43,29 @@ var ( version = "1.0.0-rc2" ) -func NewNFSdriver(nodeID, endpoint string) *nfsDriver { +func NewNFSdriver(nodeID, endpoint, controllerPlugin string) *nfsDriver { glog.Infof("Driver: %v version: %v", driverName, version) n := &nfsDriver{ - name: driverName, - version: version, - nodeID: nodeID, - endpoint: endpoint, + name: driverName, + version: version, + nodeID: nodeID, + endpoint: endpoint, + controllerPlugin: controllerPlugin, } n.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) - // NFS plugin does not support ControllerServiceCapability now. - // If support is added, it should set to appropriate - // ControllerServiceCapability RPC types. - n.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN}) + glog.Infof("controllerPlugin: %s", n.controllerPlugin) + glog.Infof("CreateVolume: %v, DeleteVolume: %v", isSupported(n.controllerPlugin, "CreateVolume"), isSupported(n.controllerPlugin, "DeleteVolume")) + createVolume, _ := lookupSymbol(n.controllerPlugin, "CreateVolume") + deleteVolume, _ := lookupSymbol(n.controllerPlugin, "DeleteVolume") + glog.Infof("CreateVolume: %v, DeleteVolume: %v", createVolume, deleteVolume) + + if isSupported(n.controllerPlugin, "CreateVolume") && isSupported(n.controllerPlugin, "DeleteVolume") { + n.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME}) + } else { + n.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN}) + } return n } diff --git a/release-tools/build.make b/release-tools/build.make index 8ca0b2c2f..619b54295 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -62,7 +62,9 @@ endif build-%: mkdir -p bin - CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* +# CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* + CGO_ENABLED=1 GOOS=linux go build -a -ldflags '-X main.version=$(REV)' -o ./bin/$* ./cmd/$* + CGO_ENABLED=1 GOOS=linux go build -tags simplenfs -buildmode=plugin -o simplenfs/bin/plugin.so simplenfs/plugin.go container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . @@ -90,7 +92,7 @@ container: $(CMDS:%=container-%) push: $(CMDS:%=push-%) clean: - -rm -rf bin + -rm -rf bin simplenfs/bin test: diff --git a/simplenfs/plugin.go b/simplenfs/plugin.go new file mode 100644 index 000000000..82bc4e727 --- /dev/null +++ b/simplenfs/plugin.go @@ -0,0 +1,206 @@ +// +build simplenfs + +package main + +import ( + "encoding/base64" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/glog" + "github.com/kubernetes-csi/csi-driver-nfs/pkg/nfs" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/pkg/util/mount" +) + +const ( + mountPathBase = "/csi-nfs-volume" +) + +func CreateVolume(cs *nfs.ControllerServer, ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + glog.Infof("plugin.CreateVolume called") + var volSize int64 + if req.GetCapacityRange() != nil { + volSize = req.GetCapacityRange().GetRequiredBytes() + } + volInfo := volumeInfo{req.GetParameters()["server"], req.GetParameters()["rootpath"], req.GetName()} + volID, err := encodeVolID(volInfo) + if err != nil { + glog.Warningf("encodeVolID for volInfo %v failed: %v", volInfo, err) + return nil, status.Error(codes.Internal, err.Error()) + } + + // Create /csi-nfs-volume/{UUID}/ directory and mount nfs rootpath to it + mountPath := filepath.Join(mountPathBase, string(uuid.NewUUID())) + if err := setupMountPath(mountPath, volInfo.server, volInfo.rootpath); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // Unmount nfs rootpath from /csi-nfs-volume/{UUID}/{volID} directory and delete the directory + defer teardownMountPath(mountPath) + + // Create directory in nfs rootpath by creating directory /csi-nfs-volume/{UUID}/{volID} + fullPath := filepath.Join(mountPath, volID) + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + glog.V(4).Infof("creating path %s", fullPath) + if err := os.MkdirAll(fullPath, 0777); err != nil { + return nil, errors.New("unable to create directory to create volume: " + err.Error()) + } + os.Chmod(fullPath, 0777) + } + + // Add share:{rootPath}/{volID} to volumeContext + volContext := req.GetParameters() + volContext["share"] = filepath.Join(volInfo.rootpath, volID) + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: volID, + CapacityBytes: volSize, + VolumeContext: volContext, + }, + }, nil +} + +func DeleteVolume(cs *nfs.ControllerServer, ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + glog.Infof("plugin.DeleteVolume called") + volumeID := req.GetVolumeId() + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "Empty volume ID in request") + } + glog.Infof("volumeID: %s", volumeID) + + volInfo, err := decodeVolID(volumeID) + if err != nil { + glog.Warningf("decodeVolID for volumeID %s failed: %v", volumeID, err) + return nil, status.Error(codes.Internal, err.Error()) + } + + // Create /csi-nfs-volume/{UUID}/ directory and mount nfs rootpath to it + mountPath := filepath.Join(mountPathBase, string(uuid.NewUUID())) + if err := setupMountPath(mountPath, volInfo.server, volInfo.rootpath); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // Unmount nfs rootpath from /csi-nfs-volume/{UUID}/{volID} directory and delete the directory + defer teardownMountPath(mountPath) + + // Delete directory in nfs rootpath by deleting directory /csi-nfs-volume/{UUID}/{volID} + fullPath := filepath.Join(mountPath, volumeID) + glog.V(4).Infof("creating path %s", fullPath) + + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + glog.Warningf("path %s does not exist, deletion skipped", fullPath) + return &csi.DeleteVolumeResponse{}, nil + } + if err := os.RemoveAll(fullPath); err != nil { + glog.Warningf("Failed to remove %s: %v", fullPath, err) + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.DeleteVolumeResponse{}, nil +} + +func ValidateVolumeCapabilities(cs *nfs.ControllerServer, ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + if req.GetVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Empty volume ID in request") + } + + if len(req.VolumeCapabilities) == 0 { + return nil, status.Error(codes.InvalidArgument, "Empty volume capabilities in request") + } + + return &csi.ValidateVolumeCapabilitiesResponse{ + Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ + VolumeCapabilities: req.VolumeCapabilities, + }, + }, nil +} + +func setupMountPath(mountPath string, server string, rootpath string) error { + // Create mountPath /csi-nfs-volume/{UUID} + if err := os.MkdirAll(mountPath, 0750); err != nil { + glog.Warningf("Failed to create mountPath %s: %v", mountPath, err) + return err + } + + // Mount nfs rootpath to mountPath /csi-nfs-volume/{UUID} + source := fmt.Sprintf("%s:%s", server, rootpath) + + mounter := mount.New("") + if err := mounter.Mount(source, mountPath, "nfs", []string{"nolock"}); err != nil { + glog.Warningf("Failed to mount source %s to mountPath %s: %v", source, mountPath, err) + return err + } + + return nil +} + +func teardownMountPath(mountPath string) error { + // Unmount nfs rootpath from mountPath /csi-nfs-volume/{UUID} and delete the path + if err := mount.CleanupMountPoint(mountPath, mount.New(""), false); err != nil { + glog.Warningf("Failed to cleanup mountPath %s: %v", mountPath, err) + return err + } + + return nil +} + +type volumeInfo struct { + server string + rootpath string + volID string +} + +func encodeVolID(vol volumeInfo) (string, error) { + if len(vol.server) == 0 { + return "", fmt.Errorf("Server information in VolumeInfo shouldn't be empty: %v", vol) + } + + if len(vol.rootpath) == 0 { + return "", fmt.Errorf("Rootpath information in VolumeInfo shouldn't be empty: %v", vol) + } + + if len(vol.volID) == 0 { + return "", fmt.Errorf("volID information in VolumeInfo shouldn't be empty: %v", vol) + } + + encServer := strings.ReplaceAll(base64.RawStdEncoding.EncodeToString([]byte(vol.server)), "/", "-") + encRootpath := strings.ReplaceAll(base64.RawStdEncoding.EncodeToString([]byte(vol.rootpath)), "/", "-") + encVolID := strings.ReplaceAll(base64.RawStdEncoding.EncodeToString([]byte(vol.volID)), "/", "-") + return strings.Join([]string{encServer, encRootpath, encVolID}, "_"), nil +} + +func decodeVolID(volID string) (*volumeInfo, error) { + var volInfo volumeInfo + volIDs := strings.SplitN(volID, "_", 3) + + if len(volIDs) != 3 { + return nil, fmt.Errorf("Failed to decode information from %s: not enough fields", volID) + } + + serverByte, err := base64.RawStdEncoding.DecodeString(strings.ReplaceAll(volIDs[0], "-", "/")) + if err != nil { + return nil, fmt.Errorf("Failed to decode server information from %s: %v", volID, err) + } + volInfo.server = string(serverByte) + + rootpathByte, err := base64.RawStdEncoding.DecodeString(strings.ReplaceAll(volIDs[1], "-", "/")) + if err != nil { + return nil, fmt.Errorf("Failed to decode rootpath information from %s: %v", volID, err) + } + volInfo.rootpath = string(rootpathByte) + + volIDByte, err := base64.RawStdEncoding.DecodeString(strings.ReplaceAll(volIDs[2], "-", "/")) + if err != nil { + return nil, fmt.Errorf("Failed to decode volID information from %s: %v", volID, err) + } + volInfo.volID = string(volIDByte) + + return &volInfo, nil +} From 3acab71987fa26e29c13972d9705bedadc8827d6 Mon Sep 17 00:00:00 2001 From: Masaki Kimura Date: Mon, 24 Jun 2019 21:52:53 +0000 Subject: [PATCH 2/2] Improve plugin to be loaded as interface --- cmd/nfsplugin/main.go | 6 ++- pkg/nfs/controllerserver.go | 82 +++++++++++----------------- pkg/nfs/nfs.go | 103 ++++++++++++++++++++++++++++-------- pkg/nfs/types.go | 58 ++++++++++++++++++++ simplenfs/plugin.go | 18 +++++-- 5 files changed, 190 insertions(+), 77 deletions(-) create mode 100644 pkg/nfs/types.go diff --git a/cmd/nfsplugin/main.go b/cmd/nfsplugin/main.go index ae144f338..390cfab2f 100644 --- a/cmd/nfsplugin/main.go +++ b/cmd/nfsplugin/main.go @@ -68,6 +68,10 @@ func main() { } func handle() { - d := nfs.NewNFSdriver(nodeID, endpoint, controllerPlugin) + d, err := nfs.NewNFSdriver(nodeID, endpoint, controllerPlugin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err.Error()) + os.Exit(1) + } d.Run() } diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index 7c205e096..e3d6b0150 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -1,10 +1,7 @@ package nfs import ( - "plugin" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/glog" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -14,104 +11,85 @@ type ControllerServer struct { Driver *nfsDriver } -func isSupported(pluginName string, symbolName string) bool { - symbol, err := lookupSymbol(pluginName, symbolName) - return err == nil && symbol != nil -} - -func lookupSymbol(pluginName string, symbolName string) (interface{}, error) { - if pluginName != "" { - plug, err := plugin.Open(pluginName) - if err != nil { - glog.Infof("Failed to load plugin: %s error: %v", pluginName, err) - return nil, err - } - symbol, err := plug.Lookup(symbolName) - if err != nil { - glog.Infof("Failed to lookup symbol: %s error: %v", symbolName, err) - return nil, err - } - return symbol, nil - } - return nil, nil -} - func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - glog.Infof("CreateVolume called") - symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "CreateVolume") - if err == nil && symbol != nil { - createVolume, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error)) - if ok { - return createVolume(cs, ctx, req) - } + if plug, ok := cs.Driver.csPlugin.(CreateDeleteVolumeControllerPlugin); ok { + return plug.CreateVolume(ctx, cs, req) } - return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { - glog.Infof("DeleteVolume called") - symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "DeleteVolume") - if err == nil && symbol != nil { - deleteVolume, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error)) - if ok { - return deleteVolume(cs, ctx, req) - } + if plug, ok := cs.Driver.csPlugin.(CreateDeleteVolumeControllerPlugin); ok { + return plug.DeleteVolume(ctx, cs, req) } - return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + if plug, ok := cs.Driver.csPlugin.(PublishUnpublishVolumeControllerPlugin); ok { + return plug.ControllerPublishVolume(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + if plug, ok := cs.Driver.csPlugin.(PublishUnpublishVolumeControllerPlugin); ok { + return plug.ControllerUnpublishVolume(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - symbol, err := lookupSymbol(cs.Driver.controllerPlugin, "ValidateVolumeCapabilities") - if err == nil && symbol != nil { - validateVolumeCapabilities, ok := symbol.(func(cs *ControllerServer, ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error)) - if ok { - return validateVolumeCapabilities(cs, ctx, req) - } + if plug, ok := cs.Driver.csPlugin.(ControllerPlugin); ok { + return plug.ValidateVolumeCapabilities(ctx, cs, req) } - return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { + if plug, ok := cs.Driver.csPlugin.(ListVolumesControllerPlugin); ok { + return plug.ListVolumes(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { + if plug, ok := cs.Driver.csPlugin.(GetCapacityControllerPlugin); ok { + return plug.GetCapacity(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } -// ControllerGetCapabilities implements the default GRPC callout. -// Default supports all capabilities func (cs *ControllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - glog.V(5).Infof("Using default ControllerGetCapabilities") - return &csi.ControllerGetCapabilitiesResponse{ Capabilities: cs.Driver.cscap, }, nil } func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + if plug, ok := cs.Driver.csPlugin.(SnapshotControllerPlugin); ok { + return plug.CreateSnapshot(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + if plug, ok := cs.Driver.csPlugin.(SnapshotControllerPlugin); ok { + return plug.DeleteSnapshot(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + if plug, ok := cs.Driver.csPlugin.(ListSnapshotControllerPlugin); ok { + return plug.ListSnapshots(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + if plug, ok := cs.Driver.csPlugin.(ExpandVolumeControllerPlugin); ok { + return plug.ControllerExpandVolume(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index c9609caaf..1f2f76714 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -17,6 +17,9 @@ limitations under the License. package nfs import ( + "fmt" + "plugin" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/glog" ) @@ -26,48 +29,108 @@ type nfsDriver struct { nodeID string version string - endpoint string - controllerPlugin string + endpoint string //ids *identityServer ns *nodeServer cap []*csi.VolumeCapability_AccessMode cscap []*csi.ControllerServiceCapability + + csPlugin interface{} } const ( - driverName = "csi-nfsplugin" + driverName = "csi-nfsplugin" + pluginSymbolName = "NfsPlugin" ) var ( version = "1.0.0-rc2" ) -func NewNFSdriver(nodeID, endpoint, controllerPlugin string) *nfsDriver { +func loadControllerPlugin(pluginName string) (interface{}, []csi.ControllerServiceCapability_RPC_Type, error) { + csc := []csi.ControllerServiceCapability_RPC_Type{} + + if pluginName == "" { + csc = append(csc, csi.ControllerServiceCapability_RPC_UNKNOWN) + return nil, csc, nil + } + + plug, err := plugin.Open(pluginName) + if err != nil { + glog.Infof("Failed to load plugin: %s error: %v", pluginName, err) + return nil, csc, err + } + + csPlugin, err := plug.Lookup(pluginSymbolName) + if err != nil { + glog.Infof("Failed to lookup csPlugin: %s error: %v", pluginSymbolName, err) + return nil, csc, err + } + + // Check if csPlugin implements each capability and add it to implenentation + if _, ok := csPlugin.(ControllerPlugin); !ok { + glog.Infof("Plugin doesn't implement mandatory methods for controller") + return nil, csc, fmt.Errorf("Plugin doesn't implement mandatory methods for controller") + } + + if _, ok := csPlugin.(CreateDeleteVolumeControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME) + } + + if _, ok := csPlugin.(PublishUnpublishVolumeControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME) + } + + if _, ok := csPlugin.(ListVolumesControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_LIST_VOLUMES) + } + + if _, ok := csPlugin.(GetCapacityControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_GET_CAPACITY) + } + + if _, ok := csPlugin.(SnapshotControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT) + } + + if _, ok := csPlugin.(ListSnapshotControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS) + } + + if _, ok := csPlugin.(ExpandVolumeControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_EXPAND_VOLUME) + } + + // TODO: Need to handle clone volume and publish read only capability? + + if len(csc) == 0 { + csc = append(csc, csi.ControllerServiceCapability_RPC_UNKNOWN) + } + + return csPlugin, csc, nil +} + +func NewNFSdriver(nodeID, endpoint, controllerPlugin string) (*nfsDriver, error) { glog.Infof("Driver: %v version: %v", driverName, version) n := &nfsDriver{ - name: driverName, - version: version, - nodeID: nodeID, - endpoint: endpoint, - controllerPlugin: controllerPlugin, + name: driverName, + version: version, + nodeID: nodeID, + endpoint: endpoint, } n.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) - glog.Infof("controllerPlugin: %s", n.controllerPlugin) - glog.Infof("CreateVolume: %v, DeleteVolume: %v", isSupported(n.controllerPlugin, "CreateVolume"), isSupported(n.controllerPlugin, "DeleteVolume")) - createVolume, _ := lookupSymbol(n.controllerPlugin, "CreateVolume") - deleteVolume, _ := lookupSymbol(n.controllerPlugin, "DeleteVolume") - glog.Infof("CreateVolume: %v, DeleteVolume: %v", createVolume, deleteVolume) - if isSupported(n.controllerPlugin, "CreateVolume") && isSupported(n.controllerPlugin, "DeleteVolume") { - n.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME}) - } else { - n.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN}) + csPlugin, csc, err := loadControllerPlugin(controllerPlugin) + if err != nil { + return nil, fmt.Errorf("Failed to load plugin %s: %v", controllerPlugin, err) } + n.csPlugin = csPlugin + n.AddControllerServiceCapabilities(csc) - return n + return n, nil } func NewNodeServer(n *nfsDriver) *nodeServer { @@ -80,8 +143,6 @@ func (n *nfsDriver) Run() { s := NewNonBlockingGRPCServer() s.Start(n.endpoint, NewDefaultIdentityServer(n), - // NFS plugin has not implemented ControllerServer - // using default controllerserver. NewControllerServer(n), NewNodeServer(n)) s.Wait() diff --git a/pkg/nfs/types.go b/pkg/nfs/types.go new file mode 100644 index 000000000..e15a676fa --- /dev/null +++ b/pkg/nfs/types.go @@ -0,0 +1,58 @@ +package nfs + +import ( + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" +) + +// ControllerPlugin is an interface for controller that implements minimum set of controller methods +type ControllerPlugin interface { + // TODO: Consider ControllerGetCapabilities needs to be implemented depend on plugins + //ControllerGetCapabilities(ctx context.Context, cs *ControllerServer, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) + ValidateVolumeCapabilities(ctx context.Context, cs *ControllerServer, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) +} + +// CreateDeleteVolumeControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME +type CreateDeleteVolumeControllerPlugin interface { + ControllerPlugin + CreateVolume(ctx context.Context, cs *ControllerServer, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) + DeleteVolume(ctx context.Context, cs *ControllerServer, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) +} + +// PublishUnpublishVolumeControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME +type PublishUnpublishVolumeControllerPlugin interface { + ControllerPlugin + ControllerPublishVolume(ctx context.Context, cs *ControllerServer, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) + ControllerUnpublishVolume(ctx context.Context, cs *ControllerServer, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) +} + +// ListVolumesControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_LIST_VOLUMES +type ListVolumesControllerPlugin interface { + ControllerPlugin + ListVolumes(ctx context.Context, cs *ControllerServer, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) +} + +// GetCapacityControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_GET_CAPACITY +type GetCapacityControllerPlugin interface { + ControllerPlugin + GetCapacity(ctx context.Context, cs *ControllerServer, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) +} + +// SnapshotControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT +type SnapshotControllerPlugin interface { + ControllerPlugin + CreateSnapshot(ctx context.Context, cs *ControllerServer, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) + DeleteSnapshot(ctx context.Context, cs *ControllerServer, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) +} + +// ListSnapshotControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS +type ListSnapshotControllerPlugin interface { + ControllerPlugin + ListSnapshots(ctx context.Context, cs *ControllerServer, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) +} + +// ExpandVolumeControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_EXPAND_VOLUME +type ExpandVolumeControllerPlugin interface { + ControllerPlugin + ControllerExpandVolume(ctx context.Context, cs *ControllerServer, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) +} diff --git a/simplenfs/plugin.go b/simplenfs/plugin.go index 82bc4e727..7ae34df05 100644 --- a/simplenfs/plugin.go +++ b/simplenfs/plugin.go @@ -24,7 +24,17 @@ const ( mountPathBase = "/csi-nfs-volume" ) -func CreateVolume(cs *nfs.ControllerServer, ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { +// csPlugin is an implementation of ControllerPlugin +type csPlugin struct { + name string +} + +var _ nfs.ControllerPlugin = csPlugin{} +var _ nfs.CreateDeleteVolumeControllerPlugin = csPlugin{} +var NfsPlugin = csPlugin{"NfsPlugin"} + +// CreateVolume is an implenetaiton that is required by CreateDeleteVolumeControllerPlugin interface +func (p csPlugin) CreateVolume(ctx context.Context, cs *nfs.ControllerServer, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { glog.Infof("plugin.CreateVolume called") var volSize int64 if req.GetCapacityRange() != nil { @@ -68,7 +78,8 @@ func CreateVolume(cs *nfs.ControllerServer, ctx context.Context, req *csi.Create }, nil } -func DeleteVolume(cs *nfs.ControllerServer, ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { +// DeleteVolume is an implenetaiton that is required by CreateDeleteVolumeControllerPlugin interface +func (p csPlugin) DeleteVolume(ctx context.Context, cs *nfs.ControllerServer, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { glog.Infof("plugin.DeleteVolume called") volumeID := req.GetVolumeId() if volumeID == "" { @@ -106,7 +117,8 @@ func DeleteVolume(cs *nfs.ControllerServer, ctx context.Context, req *csi.Delete return &csi.DeleteVolumeResponse{}, nil } -func ValidateVolumeCapabilities(cs *nfs.ControllerServer, ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { +// ValidateVolumeCapabilities is an implenetaiton that is required by ControllerPlugin interface +func (p csPlugin) ValidateVolumeCapabilities(ctx context.Context, cs *nfs.ControllerServer, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { if req.GetVolumeId() == "" { return nil, status.Error(codes.InvalidArgument, "Empty volume ID in request") }