diff --git a/.gitignore b/.gitignore index 5e56e040e..ebe27f1ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /bin +simplenfs/bin diff --git a/Dockerfile b/Dockerfile index 13f5d2485..3ca6a54cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM centos:7.4.1708 # Copy nfsplugin from build _output directory COPY bin/nfsplugin /nfsplugin +RUN mkdir -p /simplenfs/bin +COPY simplenfs/bin/plugin.so /simplenfs/plugin.so RUN yum -y install nfs-utils && yum -y install epel-release && yum -y install jq && yum clean all diff --git a/cmd/nfsplugin/main.go b/cmd/nfsplugin/main.go index f757df0b0..390cfab2f 100644 --- a/cmd/nfsplugin/main.go +++ b/cmd/nfsplugin/main.go @@ -27,8 +27,9 @@ import ( ) var ( - endpoint string - nodeID string + endpoint string + nodeID string + controllerPlugin string ) func init() { @@ -55,6 +56,8 @@ func main() { cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint") cmd.MarkPersistentFlagRequired("endpoint") + cmd.PersistentFlags().StringVar(&controllerPlugin, "controllerPlugin", "", "Controller plugin") + cmd.ParseFlags(os.Args[1:]) if err := cmd.Execute(); err != nil { fmt.Fprintf(os.Stderr, "%s", err.Error()) @@ -65,6 +68,10 @@ func main() { } func handle() { - d := nfs.NewNFSdriver(nodeID, endpoint) + d, err := nfs.NewNFSdriver(nodeID, endpoint, controllerPlugin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err.Error()) + os.Exit(1) + } d.Run() } diff --git a/deploy/kubernetes-simplenfs/csi-attacher-nfsplugin.yaml b/deploy/kubernetes-simplenfs/csi-attacher-nfsplugin.yaml new file mode 100644 index 000000000..c53c35ab4 --- /dev/null +++ b/deploy/kubernetes-simplenfs/csi-attacher-nfsplugin.yaml @@ -0,0 +1,79 @@ +# This YAML file contains attacher & csi driver API objects that are necessary +# to run external CSI attacher for nfs + +kind: Service +apiVersion: v1 +metadata: + name: csi-attacher-nfsplugin + labels: + app: csi-attacher-nfsplugin +spec: + selector: + app: csi-attacher-nfsplugin + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: csi-attacher-nfsplugin +spec: + serviceName: "csi-attacher" + replicas: 1 + template: + metadata: + labels: + app: csi-attacher-nfsplugin + spec: + serviceAccount: csi-attacher + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.1 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: unix:///csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: nfs + image: nfsplugin:latest + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--controllerPlugin=/simplenfs/plugin.so" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://plugin/csi.sock + imagePullPolicy: "IfNotPresent" + securityContext: + capabilities: + add: + - SYS_ADMIN + volumeMounts: + - name: socket-dir + mountPath: /plugin + volumes: + - name: socket-dir + emptyDir: diff --git a/deploy/kubernetes-simplenfs/csi-attacher-rbac.yaml b/deploy/kubernetes-simplenfs/csi-attacher-rbac.yaml new file mode 100644 index 000000000..8073f1227 --- /dev/null +++ b/deploy/kubernetes-simplenfs/csi-attacher-rbac.yaml @@ -0,0 +1,46 @@ +# This YAML file contains RBAC API objects that are necessary to run external +# CSI attacher for nfs flex adapter + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-attacher + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-attacher-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-role +subjects: + - kind: ServiceAccount + name: csi-attacher + namespace: default +roleRef: + kind: ClusterRole + name: external-attacher-runner + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes-simplenfs/csi-nodeplugin-nfsplugin.yaml b/deploy/kubernetes-simplenfs/csi-nodeplugin-nfsplugin.yaml new file mode 100644 index 000000000..880cc73aa --- /dev/null +++ b/deploy/kubernetes-simplenfs/csi-nodeplugin-nfsplugin.yaml @@ -0,0 +1,75 @@ +# This YAML file contains driver-registrar & csi driver nodeplugin API objects +# that are necessary to run CSI nodeplugin for nfs +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: csi-nodeplugin-nfsplugin +spec: + selector: + matchLabels: + app: csi-nodeplugin-nfsplugin + template: + metadata: + labels: + app: csi-nodeplugin-nfsplugin + spec: + serviceAccount: csi-nodeplugin + hostNetwork: true + containers: + - name: node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"] + args: + - --v=5 + - --csi-address=/plugin/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: registration-dir + mountPath: /registration + - name: nfs + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: nfsplugin:latest + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://plugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir diff --git a/deploy/kubernetes-simplenfs/csi-nodeplugin-rbac.yaml b/deploy/kubernetes-simplenfs/csi-nodeplugin-rbac.yaml new file mode 100644 index 000000000..530e067b5 --- /dev/null +++ b/deploy/kubernetes-simplenfs/csi-nodeplugin-rbac.yaml @@ -0,0 +1,34 @@ +# This YAML defines all API objects to create RBAC roles for CSI node plugin +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nodeplugin + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin +subjects: + - kind: ServiceAccount + name: csi-nodeplugin + namespace: default +roleRef: + kind: ClusterRole + name: csi-nodeplugin + apiGroup: rbac.authorization.k8s.io diff --git a/examples/kubernetes-simplenfs/nginx.yaml b/examples/kubernetes-simplenfs/nginx.yaml new file mode 100644 index 000000000..2ce222581 --- /dev/null +++ b/examples/kubernetes-simplenfs/nginx.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-simplenfsplugin +spec: + containers: + - image: maersk/nginx + imagePullPolicy: Always + name: nginx + ports: + - containerPort: 80 + protocol: TCP + volumeMounts: + - mountPath: /var/www + name: data-simplenfsplugin + volumes: + - name: data-simplenfsplugin + persistentVolumeClaim: + claimName: data-simplenfsplugin diff --git a/examples/kubernetes-simplenfs/pvc.yaml b/examples/kubernetes-simplenfs/pvc.yaml new file mode 100644 index 000000000..244eab3bf --- /dev/null +++ b/examples/kubernetes-simplenfs/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-simplenfsplugin +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: csi-simplenfs-sc diff --git a/examples/kubernetes-simplenfs/storageclass.yaml b/examples/kubernetes-simplenfs/storageclass.yaml new file mode 100644 index 000000000..6a561d600 --- /dev/null +++ b/examples/kubernetes-simplenfs/storageclass.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-simplenfs-sc +provisioner: csi-nfsplugin +parameters: + server: 192.168.122.57 + rootpath: /exports +reclaimPolicy: Delete diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index 0bbbb4235..e3d6b0150 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -2,7 +2,6 @@ package nfs import ( "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/glog" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -13,55 +12,84 @@ type ControllerServer struct { } func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + if plug, ok := cs.Driver.csPlugin.(CreateDeleteVolumeControllerPlugin); ok { + return plug.CreateVolume(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + if plug, ok := cs.Driver.csPlugin.(CreateDeleteVolumeControllerPlugin); ok { + return plug.DeleteVolume(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + if plug, ok := cs.Driver.csPlugin.(PublishUnpublishVolumeControllerPlugin); ok { + return plug.ControllerPublishVolume(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + if plug, ok := cs.Driver.csPlugin.(PublishUnpublishVolumeControllerPlugin); ok { + return plug.ControllerUnpublishVolume(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + if plug, ok := cs.Driver.csPlugin.(ControllerPlugin); ok { + return plug.ValidateVolumeCapabilities(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { + if plug, ok := cs.Driver.csPlugin.(ListVolumesControllerPlugin); ok { + return plug.ListVolumes(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { + if plug, ok := cs.Driver.csPlugin.(GetCapacityControllerPlugin); ok { + return plug.GetCapacity(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } -// ControllerGetCapabilities implements the default GRPC callout. -// Default supports all capabilities func (cs *ControllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - glog.V(5).Infof("Using default ControllerGetCapabilities") - return &csi.ControllerGetCapabilitiesResponse{ Capabilities: cs.Driver.cscap, }, nil } func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + if plug, ok := cs.Driver.csPlugin.(SnapshotControllerPlugin); ok { + return plug.CreateSnapshot(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + if plug, ok := cs.Driver.csPlugin.(SnapshotControllerPlugin); ok { + return plug.DeleteSnapshot(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + if plug, ok := cs.Driver.csPlugin.(ListSnapshotControllerPlugin); ok { + return plug.ListSnapshots(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + if plug, ok := cs.Driver.csPlugin.(ExpandVolumeControllerPlugin); ok { + return plug.ControllerExpandVolume(ctx, cs, req) + } return nil, status.Error(codes.Unimplemented, "") } diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index c5378337c..1f2f76714 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -17,6 +17,9 @@ limitations under the License. package nfs import ( + "fmt" + "plugin" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/glog" ) @@ -32,17 +35,83 @@ type nfsDriver struct { ns *nodeServer cap []*csi.VolumeCapability_AccessMode cscap []*csi.ControllerServiceCapability + + csPlugin interface{} } const ( - driverName = "csi-nfsplugin" + driverName = "csi-nfsplugin" + pluginSymbolName = "NfsPlugin" ) var ( version = "1.0.0-rc2" ) -func NewNFSdriver(nodeID, endpoint string) *nfsDriver { +func loadControllerPlugin(pluginName string) (interface{}, []csi.ControllerServiceCapability_RPC_Type, error) { + csc := []csi.ControllerServiceCapability_RPC_Type{} + + if pluginName == "" { + csc = append(csc, csi.ControllerServiceCapability_RPC_UNKNOWN) + return nil, csc, nil + } + + plug, err := plugin.Open(pluginName) + if err != nil { + glog.Infof("Failed to load plugin: %s error: %v", pluginName, err) + return nil, csc, err + } + + csPlugin, err := plug.Lookup(pluginSymbolName) + if err != nil { + glog.Infof("Failed to lookup csPlugin: %s error: %v", pluginSymbolName, err) + return nil, csc, err + } + + // Check if csPlugin implements each capability and add it to implenentation + if _, ok := csPlugin.(ControllerPlugin); !ok { + glog.Infof("Plugin doesn't implement mandatory methods for controller") + return nil, csc, fmt.Errorf("Plugin doesn't implement mandatory methods for controller") + } + + if _, ok := csPlugin.(CreateDeleteVolumeControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME) + } + + if _, ok := csPlugin.(PublishUnpublishVolumeControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME) + } + + if _, ok := csPlugin.(ListVolumesControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_LIST_VOLUMES) + } + + if _, ok := csPlugin.(GetCapacityControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_GET_CAPACITY) + } + + if _, ok := csPlugin.(SnapshotControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT) + } + + if _, ok := csPlugin.(ListSnapshotControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS) + } + + if _, ok := csPlugin.(ExpandVolumeControllerPlugin); ok { + csc = append(csc, csi.ControllerServiceCapability_RPC_EXPAND_VOLUME) + } + + // TODO: Need to handle clone volume and publish read only capability? + + if len(csc) == 0 { + csc = append(csc, csi.ControllerServiceCapability_RPC_UNKNOWN) + } + + return csPlugin, csc, nil +} + +func NewNFSdriver(nodeID, endpoint, controllerPlugin string) (*nfsDriver, error) { glog.Infof("Driver: %v version: %v", driverName, version) n := &nfsDriver{ @@ -53,12 +122,15 @@ func NewNFSdriver(nodeID, endpoint string) *nfsDriver { } n.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) - // NFS plugin does not support ControllerServiceCapability now. - // If support is added, it should set to appropriate - // ControllerServiceCapability RPC types. - n.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN}) - return n + csPlugin, csc, err := loadControllerPlugin(controllerPlugin) + if err != nil { + return nil, fmt.Errorf("Failed to load plugin %s: %v", controllerPlugin, err) + } + n.csPlugin = csPlugin + n.AddControllerServiceCapabilities(csc) + + return n, nil } func NewNodeServer(n *nfsDriver) *nodeServer { @@ -71,8 +143,6 @@ func (n *nfsDriver) Run() { s := NewNonBlockingGRPCServer() s.Start(n.endpoint, NewDefaultIdentityServer(n), - // NFS plugin has not implemented ControllerServer - // using default controllerserver. NewControllerServer(n), NewNodeServer(n)) s.Wait() diff --git a/pkg/nfs/types.go b/pkg/nfs/types.go new file mode 100644 index 000000000..e15a676fa --- /dev/null +++ b/pkg/nfs/types.go @@ -0,0 +1,58 @@ +package nfs + +import ( + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" +) + +// ControllerPlugin is an interface for controller that implements minimum set of controller methods +type ControllerPlugin interface { + // TODO: Consider ControllerGetCapabilities needs to be implemented depend on plugins + //ControllerGetCapabilities(ctx context.Context, cs *ControllerServer, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) + ValidateVolumeCapabilities(ctx context.Context, cs *ControllerServer, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) +} + +// CreateDeleteVolumeControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME +type CreateDeleteVolumeControllerPlugin interface { + ControllerPlugin + CreateVolume(ctx context.Context, cs *ControllerServer, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) + DeleteVolume(ctx context.Context, cs *ControllerServer, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) +} + +// PublishUnpublishVolumeControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME +type PublishUnpublishVolumeControllerPlugin interface { + ControllerPlugin + ControllerPublishVolume(ctx context.Context, cs *ControllerServer, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) + ControllerUnpublishVolume(ctx context.Context, cs *ControllerServer, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) +} + +// ListVolumesControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_LIST_VOLUMES +type ListVolumesControllerPlugin interface { + ControllerPlugin + ListVolumes(ctx context.Context, cs *ControllerServer, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) +} + +// GetCapacityControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_GET_CAPACITY +type GetCapacityControllerPlugin interface { + ControllerPlugin + GetCapacity(ctx context.Context, cs *ControllerServer, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) +} + +// SnapshotControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT +type SnapshotControllerPlugin interface { + ControllerPlugin + CreateSnapshot(ctx context.Context, cs *ControllerServer, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) + DeleteSnapshot(ctx context.Context, cs *ControllerServer, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) +} + +// ListSnapshotControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS +type ListSnapshotControllerPlugin interface { + ControllerPlugin + ListSnapshots(ctx context.Context, cs *ControllerServer, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) +} + +// ExpandVolumeControllerPlugin is an interface for controller that implements csi.ControllerServiceCapability_RPC_EXPAND_VOLUME +type ExpandVolumeControllerPlugin interface { + ControllerPlugin + ControllerExpandVolume(ctx context.Context, cs *ControllerServer, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) +} diff --git a/release-tools/build.make b/release-tools/build.make index 8ca0b2c2f..619b54295 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -62,7 +62,9 @@ endif build-%: mkdir -p bin - CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* +# CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* + CGO_ENABLED=1 GOOS=linux go build -a -ldflags '-X main.version=$(REV)' -o ./bin/$* ./cmd/$* + CGO_ENABLED=1 GOOS=linux go build -tags simplenfs -buildmode=plugin -o simplenfs/bin/plugin.so simplenfs/plugin.go container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . @@ -90,7 +92,7 @@ container: $(CMDS:%=container-%) push: $(CMDS:%=push-%) clean: - -rm -rf bin + -rm -rf bin simplenfs/bin test: diff --git a/simplenfs/plugin.go b/simplenfs/plugin.go new file mode 100644 index 000000000..7ae34df05 --- /dev/null +++ b/simplenfs/plugin.go @@ -0,0 +1,218 @@ +// +build simplenfs + +package main + +import ( + "encoding/base64" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/glog" + "github.com/kubernetes-csi/csi-driver-nfs/pkg/nfs" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/pkg/util/mount" +) + +const ( + mountPathBase = "/csi-nfs-volume" +) + +// csPlugin is an implementation of ControllerPlugin +type csPlugin struct { + name string +} + +var _ nfs.ControllerPlugin = csPlugin{} +var _ nfs.CreateDeleteVolumeControllerPlugin = csPlugin{} +var NfsPlugin = csPlugin{"NfsPlugin"} + +// CreateVolume is an implenetaiton that is required by CreateDeleteVolumeControllerPlugin interface +func (p csPlugin) CreateVolume(ctx context.Context, cs *nfs.ControllerServer, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + glog.Infof("plugin.CreateVolume called") + var volSize int64 + if req.GetCapacityRange() != nil { + volSize = req.GetCapacityRange().GetRequiredBytes() + } + volInfo := volumeInfo{req.GetParameters()["server"], req.GetParameters()["rootpath"], req.GetName()} + volID, err := encodeVolID(volInfo) + if err != nil { + glog.Warningf("encodeVolID for volInfo %v failed: %v", volInfo, err) + return nil, status.Error(codes.Internal, err.Error()) + } + + // Create /csi-nfs-volume/{UUID}/ directory and mount nfs rootpath to it + mountPath := filepath.Join(mountPathBase, string(uuid.NewUUID())) + if err := setupMountPath(mountPath, volInfo.server, volInfo.rootpath); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // Unmount nfs rootpath from /csi-nfs-volume/{UUID}/{volID} directory and delete the directory + defer teardownMountPath(mountPath) + + // Create directory in nfs rootpath by creating directory /csi-nfs-volume/{UUID}/{volID} + fullPath := filepath.Join(mountPath, volID) + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + glog.V(4).Infof("creating path %s", fullPath) + if err := os.MkdirAll(fullPath, 0777); err != nil { + return nil, errors.New("unable to create directory to create volume: " + err.Error()) + } + os.Chmod(fullPath, 0777) + } + + // Add share:{rootPath}/{volID} to volumeContext + volContext := req.GetParameters() + volContext["share"] = filepath.Join(volInfo.rootpath, volID) + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: volID, + CapacityBytes: volSize, + VolumeContext: volContext, + }, + }, nil +} + +// DeleteVolume is an implenetaiton that is required by CreateDeleteVolumeControllerPlugin interface +func (p csPlugin) DeleteVolume(ctx context.Context, cs *nfs.ControllerServer, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + glog.Infof("plugin.DeleteVolume called") + volumeID := req.GetVolumeId() + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "Empty volume ID in request") + } + glog.Infof("volumeID: %s", volumeID) + + volInfo, err := decodeVolID(volumeID) + if err != nil { + glog.Warningf("decodeVolID for volumeID %s failed: %v", volumeID, err) + return nil, status.Error(codes.Internal, err.Error()) + } + + // Create /csi-nfs-volume/{UUID}/ directory and mount nfs rootpath to it + mountPath := filepath.Join(mountPathBase, string(uuid.NewUUID())) + if err := setupMountPath(mountPath, volInfo.server, volInfo.rootpath); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // Unmount nfs rootpath from /csi-nfs-volume/{UUID}/{volID} directory and delete the directory + defer teardownMountPath(mountPath) + + // Delete directory in nfs rootpath by deleting directory /csi-nfs-volume/{UUID}/{volID} + fullPath := filepath.Join(mountPath, volumeID) + glog.V(4).Infof("creating path %s", fullPath) + + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + glog.Warningf("path %s does not exist, deletion skipped", fullPath) + return &csi.DeleteVolumeResponse{}, nil + } + if err := os.RemoveAll(fullPath); err != nil { + glog.Warningf("Failed to remove %s: %v", fullPath, err) + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.DeleteVolumeResponse{}, nil +} + +// ValidateVolumeCapabilities is an implenetaiton that is required by ControllerPlugin interface +func (p csPlugin) ValidateVolumeCapabilities(ctx context.Context, cs *nfs.ControllerServer, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + if req.GetVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Empty volume ID in request") + } + + if len(req.VolumeCapabilities) == 0 { + return nil, status.Error(codes.InvalidArgument, "Empty volume capabilities in request") + } + + return &csi.ValidateVolumeCapabilitiesResponse{ + Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ + VolumeCapabilities: req.VolumeCapabilities, + }, + }, nil +} + +func setupMountPath(mountPath string, server string, rootpath string) error { + // Create mountPath /csi-nfs-volume/{UUID} + if err := os.MkdirAll(mountPath, 0750); err != nil { + glog.Warningf("Failed to create mountPath %s: %v", mountPath, err) + return err + } + + // Mount nfs rootpath to mountPath /csi-nfs-volume/{UUID} + source := fmt.Sprintf("%s:%s", server, rootpath) + + mounter := mount.New("") + if err := mounter.Mount(source, mountPath, "nfs", []string{"nolock"}); err != nil { + glog.Warningf("Failed to mount source %s to mountPath %s: %v", source, mountPath, err) + return err + } + + return nil +} + +func teardownMountPath(mountPath string) error { + // Unmount nfs rootpath from mountPath /csi-nfs-volume/{UUID} and delete the path + if err := mount.CleanupMountPoint(mountPath, mount.New(""), false); err != nil { + glog.Warningf("Failed to cleanup mountPath %s: %v", mountPath, err) + return err + } + + return nil +} + +type volumeInfo struct { + server string + rootpath string + volID string +} + +func encodeVolID(vol volumeInfo) (string, error) { + if len(vol.server) == 0 { + return "", fmt.Errorf("Server information in VolumeInfo shouldn't be empty: %v", vol) + } + + if len(vol.rootpath) == 0 { + return "", fmt.Errorf("Rootpath information in VolumeInfo shouldn't be empty: %v", vol) + } + + if len(vol.volID) == 0 { + return "", fmt.Errorf("volID information in VolumeInfo shouldn't be empty: %v", vol) + } + + encServer := strings.ReplaceAll(base64.RawStdEncoding.EncodeToString([]byte(vol.server)), "/", "-") + encRootpath := strings.ReplaceAll(base64.RawStdEncoding.EncodeToString([]byte(vol.rootpath)), "/", "-") + encVolID := strings.ReplaceAll(base64.RawStdEncoding.EncodeToString([]byte(vol.volID)), "/", "-") + return strings.Join([]string{encServer, encRootpath, encVolID}, "_"), nil +} + +func decodeVolID(volID string) (*volumeInfo, error) { + var volInfo volumeInfo + volIDs := strings.SplitN(volID, "_", 3) + + if len(volIDs) != 3 { + return nil, fmt.Errorf("Failed to decode information from %s: not enough fields", volID) + } + + serverByte, err := base64.RawStdEncoding.DecodeString(strings.ReplaceAll(volIDs[0], "-", "/")) + if err != nil { + return nil, fmt.Errorf("Failed to decode server information from %s: %v", volID, err) + } + volInfo.server = string(serverByte) + + rootpathByte, err := base64.RawStdEncoding.DecodeString(strings.ReplaceAll(volIDs[1], "-", "/")) + if err != nil { + return nil, fmt.Errorf("Failed to decode rootpath information from %s: %v", volID, err) + } + volInfo.rootpath = string(rootpathByte) + + volIDByte, err := base64.RawStdEncoding.DecodeString(strings.ReplaceAll(volIDs[2], "-", "/")) + if err != nil { + return nil, fmt.Errorf("Failed to decode volID information from %s: %v", volID, err) + } + volInfo.volID = string(volIDByte) + + return &volInfo, nil +}