diff --git a/charts/latest/csi-driver-nfs-v0.0.0.tgz b/charts/latest/csi-driver-nfs-v0.0.0.tgz index 0ef63816c..ea01024fa 100644 Binary files a/charts/latest/csi-driver-nfs-v0.0.0.tgz and b/charts/latest/csi-driver-nfs-v0.0.0.tgz differ diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml index 255d93fe2..1f5b0c773 100644 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml @@ -63,6 +63,22 @@ spec: resources: {{- toYaml .Values.controller.resources.csiProvisioner | nindent 12 }} securityContext: readOnlyRootFilesystem: true + {{- if .Values.externalSnapshotter.enabled }} + - name: csi-snapshotter + image: "{{ .Values.image.csiSnapshotter.repository }}:{{ .Values.image.csiSnapshotter.tag }}" + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election-namespace={{ .Release.Namespace }}" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiSnapshotter.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- end }} - name: liveness-probe image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" args: diff --git a/charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml b/charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml new file mode 100644 index 000000000..01ff7a60e --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml @@ -0,0 +1,51 @@ +{{- if .Values.externalSnapshotter.enabled -}} +# This YAML file shows how to deploy the snapshot controller + +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }} + namespace: {{ .Release.Namespace }} +{{ include "nfs.labels" . | indent 2 }} + app: {{ .Values.externalSnapshotter.name }} +{{- with .Values.externalSnapshotter.labels }} +{{ . | toYaml | indent 4 }} +{{- end }} +{{- with .Values.externalSnapshotter.annotations }} + annotations: +{{ . | toYaml | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.externalSnapshotter.controller.replicas }} + selector: + matchLabels: + app: {{ .Values.externalSnapshotter.name }} + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Values.externalSnapshotter.name }} + spec: + serviceAccountName: {{ .Values.externalSnapshotter.name }} + containers: + - name: {{ .Values.externalSnapshotter.name }} + image: {{ .Values.image.externalSnapshotter.repository }}:{{ .Values.image.externalSnapshotter.tag }} + args: + - "--v=5" + - "--leader-election=true" + - "--leader-election-namespace={{ .Release.Namespace }}" + imagePullPolicy: {{ .Values.image.externalSnapshotter.pullPolicy }} +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml b/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml index bc94431d1..66c76ff72 100644 --- a/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml +++ b/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml @@ -32,6 +32,17 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] + {{- if .Values.externalSnapshotter.enabled }} + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["get", "update", "patch"] + {{- end }} - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml b/charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml new file mode 100644 index 000000000..f4fb5181b --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml @@ -0,0 +1,93 @@ +{{- if .Values.externalSnapshotter.enabled -}} +# RBAC file for the snapshot controller. +# +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.externalSnapshotter.name }} + namespace: {{ .Release.Namespace }} +{{ include "nfs.labels" . | indent 2 }} + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }}-runner +{{ include "nfs.labels" . | indent 2 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] +{{- if .Values.externalSnapshotter.enabledDistributedSnapshotting }} + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +{{- end }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }}-role +{{ include "nfs.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.externalSnapshotter.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.externalSnapshotter.name }}-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }}-leaderelection + namespace: {{ .Release.Namespace }} +{{ include "nfs.labels" . | indent 2 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }}-leaderelection + namespace: {{ .Release.Namespace }} +{{ include "nfs.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.externalSnapshotter.name }} +roleRef: + kind: Role + name: {{ .Values.externalSnapshotter.name }}-leaderelection + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml new file mode 100644 index 000000000..a4464c626 --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -0,0 +1,146 @@ +{{- if .Values.externalSnapshotter.enabled -}} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: [vsclass, vsclasses] + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying + storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its + bound VolumeSnapshot is deleted. Supported values are "Retain" and + "Delete". "Retain" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are kept. "Delete" means that + the VolumeSnapshotContent and its physical snapshot on underlying + storage system are deleted. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: [deletionPolicy, driver] + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; + use snapshot.storage.k8s.io/v1 VolumeSnapshotClass + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying + storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its + bound VolumeSnapshot is deleted. Supported values are "Retain" and + "Delete". "Retain" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are kept. "Delete" means that + the VolumeSnapshotContent and its physical snapshot on underlying + storage system are deleted. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: [deletionPolicy, driver] + type: object + served: false + storage: false + subresources: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml new file mode 100644 index 000000000..0ff8596e0 --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -0,0 +1,473 @@ +{{- if .Values.externalSnapshotter.enabled -}} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: [vsc, vscs] + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot + on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported + values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are deleted. For dynamically + provisioned snapshots, this field will automatically be filled + in by the CSI snapshotter sidecar with the "DeletionPolicy" field + defined in the corresponding VolumeSnapshotClass. For pre-existing + snapshots, users MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the CSI driver used to create + the physical snapshot on the underlying storage system. This MUST + be the same as the name returned by the CSI GetPluginName() call + for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should + be) dynamically provisioned or already exists, and just requires + a Kubernetes object representation. This field is immutable after + creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" + of a pre-existing snapshot on the underlying storage system + for which a Kubernetes object representation was (or should + be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + oneOf: + - required: [snapshotHandle] + - required: [volumeHandle] + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and + as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for + the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after + creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: [deletionPolicy, driver, source, volumeSnapshotRef] + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the CSI + snapshotter sidecar with the "creation_time" value returned from + CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this + field will be filled with the "creation_time" value returned from + the CSI "ListSnapshots" gRPC call if the driver supports it. If + not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an + int64. On Unix, the command `date +%s%N` returns the current time + in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. When restoring a volume from this snapshot, the size + of the volume MUST NOT be smaller than the restoreSize if it is + specified, otherwise the restoration will fail. If not specified, + it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: [spec] + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot + on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is + deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported + values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are deleted. For dynamically + provisioned snapshots, this field will automatically be filled + in by the CSI snapshotter sidecar with the "DeletionPolicy" field + defined in the corresponding VolumeSnapshotClass. For pre-existing + snapshots, users MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the CSI driver used to create + the physical snapshot on the underlying storage system. This MUST + be the same as the name returned by the CSI GetPluginName() call + for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should + be) dynamically provisioned or already exists, and just requires + a Kubernetes object representation. This field is immutable after + creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" + of a pre-existing snapshot on the underlying storage system + for which a Kubernetes object representation was (or should + be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and + as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for + the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after + creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: [deletionPolicy, driver, source, volumeSnapshotRef] + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the CSI + snapshotter sidecar with the "creation_time" value returned from + CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this + field will be filled with the "creation_time" value returned from + the CSI "ListSnapshots" gRPC call if the driver supports it. If + not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an + int64. On Unix, the command `date +%s%N` returns the current time + in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. When restoring a volume from this snapshot, the size + of the volume MUST NOT be smaller than the restoreSize if it is + specified, otherwise the restoration will fail. If not specified, + it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: [spec] + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml new file mode 100644 index 000000000..9447d45d2 --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml @@ -0,0 +1,387 @@ +{{- if .Values.externalSnapshotter.enabled -}} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: [vs] + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name + of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the + existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate + from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage + of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the + underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot + requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of + the PersistentVolumeClaim object representing the volume from + which a snapshot should be created. This PVC is assumed to + be in the same namespace as the VolumeSnapshot object. This + field should be set if the snapshot does not exists, and needs + to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of + a pre-existing VolumeSnapshotContent object representing an + existing volume snapshot. This field should be set if the + snapshot already exists and only needs a representation in + Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: [persistentVolumeClaimName] + - required: [volumeSnapshotContentName] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be + used. A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one + VolumeSnapshotClass exist for a given CSI Driver and more than + one have been marked as default, CreateSnapshot will fail and + generate an event. Empty string is not allowed for this field.' + type: string + required: [source] + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and + VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the + VolumeSnapshotContent object to which this VolumeSnapshot object + intends to bind to. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: To avoid possible security issues, consumers + must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot + and VolumeSnapshotContent point at each other) before using this + object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the snapshot + controller with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may + indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue + on waiting for the snapshot to be created based on the type of + error reported. The snapshot controller will keep retrying when + an error occurs during the snapshot creation. Upon success, this + error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be + used to restore a volume. In dynamic snapshot creation case, this + field will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller + with the "size_bytes" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than + the restoreSize if it is specified, otherwise the restoration + will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: [spec] + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name + of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the + existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate + from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage + of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the + underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; + use snapshot.storage.k8s.io/v1 VolumeSnapshot + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot + requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of + the PersistentVolumeClaim object representing the volume from + which a snapshot should be created. This PVC is assumed to + be in the same namespace as the VolumeSnapshot object. This + field should be set if the snapshot does not exists, and needs + to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of + a pre-existing VolumeSnapshotContent object representing an + existing volume snapshot. This field should be set if the + snapshot already exists and only needs a representation in + Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be + used. A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one + VolumeSnapshotClass exist for a given CSI Driver and more than + one have been marked as default, CreateSnapshot will fail and + generate an event. Empty string is not allowed for this field.' + type: string + required: [source] + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and + VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the + VolumeSnapshotContent object to which this VolumeSnapshot object + intends to bind to. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: To avoid possible security issues, consumers + must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot + and VolumeSnapshotContent point at each other) before using this + object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the snapshot + controller with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may + indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue + on waiting for the snapshot to be created based on the type of + error reported. The snapshot controller will keep retrying when + an error occurs during the snapshot creation. Upon success, this + error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be + used to restore a volume. In dynamic snapshot creation case, this + field will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller + with the "size_bytes" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than + the restoreSize if it is specified, otherwise the restoration + will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: [spec] + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/values.yaml b/charts/latest/csi-driver-nfs/values.yaml index 97da53e85..c621a666f 100755 --- a/charts/latest/csi-driver-nfs/values.yaml +++ b/charts/latest/csi-driver-nfs/values.yaml @@ -8,6 +8,10 @@ image: repository: registry.k8s.io/sig-storage/csi-provisioner tag: v3.3.0 pullPolicy: IfNotPresent + csiSnapshotter: + repository: registry.k8s.io/sig-storage/csi-snapshotter + tag: v6.2.1 + pullPolicy: IfNotPresent livenessProbe: repository: registry.k8s.io/sig-storage/livenessprobe tag: v2.8.0 @@ -16,6 +20,10 @@ image: repository: registry.k8s.io/sig-storage/csi-node-driver-registrar tag: v2.6.2 pullPolicy: IfNotPresent + externalSnapshotter: + repository: registry.k8s.io/sig-storage/snapshot-controller + tag: v6.1.0 + pullPolicy: IfNotPresent serviceAccount: create: true # When true, service accounts will be created for you. Set to false if you want to use your own. @@ -110,6 +118,12 @@ node: cpu: 10m memory: 20Mi +externalSnapshotter: + enabled: true + name: snapshot-controller + controller: + replicas: 1 + ## Reference to one or more secrets to be used when pulling images ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## diff --git a/deploy/csi-nfs-controller.yaml b/deploy/csi-nfs-controller.yaml index c35cc5862..05fdfb46f 100644 --- a/deploy/csi-nfs-controller.yaml +++ b/deploy/csi-nfs-controller.yaml @@ -51,6 +51,20 @@ spec: requests: cpu: 10m memory: 20Mi + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election-namespace=kube-system" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /csi - name: liveness-probe image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 args: diff --git a/deploy/csi-snapshot-controller.yaml b/deploy/csi-snapshot-controller.yaml new file mode 100644 index 000000000..cd5b86836 --- /dev/null +++ b/deploy/csi-snapshot-controller.yaml @@ -0,0 +1,40 @@ +# This YAML file shows how to deploy the snapshot controller + +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: snapshot-controller + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: snapshot-controller + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: snapshot-controller + spec: + serviceAccountName: snapshot-controller + containers: + - name: snapshot-controller + image: registry.k8s.io/sig-storage/snapshot-controller:v6.1.0 + args: + - "--v=5" + - "--leader-election=true" + imagePullPolicy: IfNotPresent diff --git a/deploy/example/snapshot-nfs-dynamic.yaml b/deploy/example/snapshot-nfs-dynamic.yaml new file mode 100644 index 000000000..7795d0681 --- /dev/null +++ b/deploy/example/snapshot-nfs-dynamic.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: test-nfs-snapshot +spec: + volumeSnapshotClassName: csi-nfs-snapclass + source: + persistentVolumeClaimName: pvc-nfs-dynamic diff --git a/deploy/example/snapshotclass-nfs.yaml b/deploy/example/snapshotclass-nfs.yaml new file mode 100644 index 000000000..d6b8e3332 --- /dev/null +++ b/deploy/example/snapshotclass-nfs.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: csi-nfs-snapclass +driver: nfs.csi.k8s.io +deletionPolicy: Delete diff --git a/deploy/install-driver.sh b/deploy/install-driver.sh index 2c4006bc1..28d2d2e9c 100755 --- a/deploy/install-driver.sh +++ b/deploy/install-driver.sh @@ -16,6 +16,13 @@ set -euo pipefail +# min version of this nfs plugin supporting CSI snapshots +snap_ver='4.3.0' + +# external-snapshotter to install for snapshots to work +snap_controller_repo='github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.2.1' +snap_crd_repo='github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.2.1' + ver="master" if [[ "$#" -gt 0 ]]; then ver="$1" @@ -33,6 +40,12 @@ if [ $ver != "master" ]; then repo="$repo/$ver" fi +# check for min supported version for snapshots or master to install external-snapshotter +if [[ "$ver" == master || "$(printf '%s\n' "$ver" "$snap_ver" | sort -V | head -n1)" = "$snap_ver" ]]; then + kubectl kustomize "$snap_crd_repo" | kubectl apply -f - + kubectl -n kube-system kustomize "$snap_controller_repo" | kubectl apply -f - +fi + echo "Installing NFS CSI driver, version: $ver ..." kubectl apply -f $repo/rbac-csi-nfs.yaml kubectl apply -f $repo/csi-nfs-driverinfo.yaml diff --git a/deploy/rbac-csi-nfs.yaml b/deploy/rbac-csi-nfs.yaml index 45c80e932..081d76e5d 100644 --- a/deploy/rbac-csi-nfs.yaml +++ b/deploy/rbac-csi-nfs.yaml @@ -26,6 +26,15 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["get", "update", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/deploy/rbac-snapshot-controller.yaml b/deploy/rbac-snapshot-controller.yaml new file mode 100644 index 000000000..2e6431bd3 --- /dev/null +++ b/deploy/rbac-snapshot-controller.yaml @@ -0,0 +1,82 @@ +# RBAC file for the snapshot controller. +# +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system +subjects: + - kind: ServiceAccount + name: snapshot-controller +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml b/deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml new file mode 100644 index 000000000..f656a4a42 --- /dev/null +++ b/deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -0,0 +1,144 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: [vsclass, vsclasses] + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying + storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its + bound VolumeSnapshot is deleted. Supported values are "Retain" and + "Delete". "Retain" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are kept. "Delete" means that + the VolumeSnapshotContent and its physical snapshot on underlying + storage system are deleted. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: [deletionPolicy, driver] + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; + use snapshot.storage.k8s.io/v1 VolumeSnapshotClass + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying + storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its + bound VolumeSnapshot is deleted. Supported values are "Retain" and + "Delete". "Retain" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are kept. "Delete" means that + the VolumeSnapshotContent and its physical snapshot on underlying + storage system are deleted. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: [deletionPolicy, driver] + type: object + served: false + storage: false + subresources: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] diff --git a/deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml b/deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml new file mode 100644 index 000000000..381466c03 --- /dev/null +++ b/deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -0,0 +1,471 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: [vsc, vscs] + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot + on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported + values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are deleted. For dynamically + provisioned snapshots, this field will automatically be filled + in by the CSI snapshotter sidecar with the "DeletionPolicy" field + defined in the corresponding VolumeSnapshotClass. For pre-existing + snapshots, users MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the CSI driver used to create + the physical snapshot on the underlying storage system. This MUST + be the same as the name returned by the CSI GetPluginName() call + for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should + be) dynamically provisioned or already exists, and just requires + a Kubernetes object representation. This field is immutable after + creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" + of a pre-existing snapshot on the underlying storage system + for which a Kubernetes object representation was (or should + be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + oneOf: + - required: [snapshotHandle] + - required: [volumeHandle] + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and + as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for + the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after + creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: [deletionPolicy, driver, source, volumeSnapshotRef] + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the CSI + snapshotter sidecar with the "creation_time" value returned from + CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this + field will be filled with the "creation_time" value returned from + the CSI "ListSnapshots" gRPC call if the driver supports it. If + not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an + int64. On Unix, the command `date +%s%N` returns the current time + in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. When restoring a volume from this snapshot, the size + of the volume MUST NOT be smaller than the restoreSize if it is + specified, otherwise the restoration will fail. If not specified, + it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: [spec] + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot + on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is + deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported + values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are deleted. For dynamically + provisioned snapshots, this field will automatically be filled + in by the CSI snapshotter sidecar with the "DeletionPolicy" field + defined in the corresponding VolumeSnapshotClass. For pre-existing + snapshots, users MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the CSI driver used to create + the physical snapshot on the underlying storage system. This MUST + be the same as the name returned by the CSI GetPluginName() call + for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should + be) dynamically provisioned or already exists, and just requires + a Kubernetes object representation. This field is immutable after + creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" + of a pre-existing snapshot on the underlying storage system + for which a Kubernetes object representation was (or should + be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and + as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for + the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after + creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: [deletionPolicy, driver, source, volumeSnapshotRef] + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the CSI + snapshotter sidecar with the "creation_time" value returned from + CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this + field will be filled with the "creation_time" value returned from + the CSI "ListSnapshots" gRPC call if the driver supports it. If + not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an + int64. On Unix, the command `date +%s%N` returns the current time + in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. When restoring a volume from this snapshot, the size + of the volume MUST NOT be smaller than the restoreSize if it is + specified, otherwise the restoration will fail. If not specified, + it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: [spec] + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] diff --git a/deploy/snapshot.storage.k8s.io_volumesnapshots.yaml b/deploy/snapshot.storage.k8s.io_volumesnapshots.yaml new file mode 100644 index 000000000..3d27cc054 --- /dev/null +++ b/deploy/snapshot.storage.k8s.io_volumesnapshots.yaml @@ -0,0 +1,385 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: [vs] + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name + of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the + existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate + from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage + of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the + underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot + requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of + the PersistentVolumeClaim object representing the volume from + which a snapshot should be created. This PVC is assumed to + be in the same namespace as the VolumeSnapshot object. This + field should be set if the snapshot does not exists, and needs + to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of + a pre-existing VolumeSnapshotContent object representing an + existing volume snapshot. This field should be set if the + snapshot already exists and only needs a representation in + Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: [persistentVolumeClaimName] + - required: [volumeSnapshotContentName] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be + used. A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one + VolumeSnapshotClass exist for a given CSI Driver and more than + one have been marked as default, CreateSnapshot will fail and + generate an event. Empty string is not allowed for this field.' + type: string + required: [source] + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and + VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the + VolumeSnapshotContent object to which this VolumeSnapshot object + intends to bind to. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: To avoid possible security issues, consumers + must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot + and VolumeSnapshotContent point at each other) before using this + object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the snapshot + controller with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may + indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue + on waiting for the snapshot to be created based on the type of + error reported. The snapshot controller will keep retrying when + an error occurs during the snapshot creation. Upon success, this + error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be + used to restore a volume. In dynamic snapshot creation case, this + field will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller + with the "size_bytes" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than + the restoreSize if it is specified, otherwise the restoration + will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: [spec] + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name + of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the + existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate + from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage + of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the + underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; + use snapshot.storage.k8s.io/v1 VolumeSnapshot + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot + requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of + the PersistentVolumeClaim object representing the volume from + which a snapshot should be created. This PVC is assumed to + be in the same namespace as the VolumeSnapshot object. This + field should be set if the snapshot does not exists, and needs + to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of + a pre-existing VolumeSnapshotContent object representing an + existing volume snapshot. This field should be set if the + snapshot already exists and only needs a representation in + Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be + used. A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one + VolumeSnapshotClass exist for a given CSI Driver and more than + one have been marked as default, CreateSnapshot will fail and + generate an event. Empty string is not allowed for this field.' + type: string + required: [source] + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and + VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the + VolumeSnapshotContent object to which this VolumeSnapshot object + intends to bind to. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: To avoid possible security issues, consumers + must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot + and VolumeSnapshotContent point at each other) before using this + object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the snapshot + controller with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may + indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue + on waiting for the snapshot to be created based on the type of + error reported. The snapshot controller will keep retrying when + an error occurs during the snapshot creation. Upon success, this + error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be + used to restore a volume. In dynamic snapshot creation case, this + field will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller + with the "size_bytes" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than + the restoreSize if it is specified, otherwise the restoration + will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: [spec] + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] diff --git a/deploy/uninstall-driver.sh b/deploy/uninstall-driver.sh index d563fc981..acd40f558 100755 --- a/deploy/uninstall-driver.sh +++ b/deploy/uninstall-driver.sh @@ -16,6 +16,10 @@ set -euo pipefail +# external-snapshotter to install for snapshots to work +snap_controller_repo='github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.2.1' +snap_crd_repo='github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.2.1' + ver="master" if [[ "$#" -gt 0 ]]; then ver="$1" @@ -34,6 +38,8 @@ if [ $ver != "master" ]; then fi echo "Uninstalling NFS driver, version: $ver ..." +# this keeps VolumeSnapshot CRDs untouched +kubectl -n kube-system kustomize "$snap_controller_repo" | kubectl delete --ignore-not-found -f - kubectl delete -f $repo/csi-nfs-controller.yaml --ignore-not-found kubectl delete -f $repo/csi-nfs-node.yaml --ignore-not-found kubectl delete -f $repo/csi-nfs-driverinfo.yaml --ignore-not-found diff --git a/go.mod b/go.mod index 5140a414f..ea4a98d05 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/stretchr/testify v1.8.0 golang.org/x/net v0.7.0 google.golang.org/grpc v1.40.0 + google.golang.org/protobuf v1.27.1 k8s.io/api v0.23.14 k8s.io/apimachinery v0.23.14 k8s.io/client-go v0.23.14 @@ -82,7 +83,6 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect - google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/hack/verify-helm-chart.sh b/hack/verify-helm-chart.sh index 174f06f77..642eec226 100755 --- a/hack/verify-helm-chart.sh +++ b/hack/verify-helm-chart.sh @@ -62,12 +62,16 @@ pip install yq --ignore-installed PyYAML # Extract images from csi-nfs-controller.yaml expected_csi_provisioner_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[0].image | head -n 1)" -expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)" -expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)" +expected_csi_snapshotter_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)" +expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)" +expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[3].image | head -n 1)" csi_provisioner_image="$(get_image_from_helm_chart "csiProvisioner")" validate_image "${expected_csi_provisioner_image}" "${csi_provisioner_image}" +csi_snapshotter_image="$(get_image_from_helm_chart "csiSnapshotter")" +validate_image "${expected_csi_snapshotter_image}" "${csi_snapshotter_image}" + liveness_probe_image="$(get_image_from_helm_chart "livenessProbe")" validate_image "${expected_liveness_probe_image}" "${liveness_probe_image}" diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index eb8c52b01..a1911ac04 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -18,6 +18,7 @@ package nfs import ( "fmt" + "io/fs" "os" "os/exec" "path/filepath" @@ -29,6 +30,7 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/klog/v2" ) @@ -59,6 +61,35 @@ type nfsVolume struct { onDelete string } +// nfsSnapshot is an internal representation of a volume snapshot +// created by the provisioner. +type nfsSnapshot struct { + // Snapshot id. + id string + // Address of the NFS server. + // Matches paramServer. + server string + // Base directory of the NFS server to create snapshots under + // Matches paramShare. + baseDir string + // Snapshot name. + uuid string + // Source volume. + src string +} + +func (snap nfsSnapshot) archiveSubPath() string { + return snap.uuid +} + +func (snap nfsSnapshot) archiveName() string { + return fmt.Sprintf("%v.tar.gz", snap.src) +} + +func (snap nfsSnapshot) archivePath() string { + return filepath.Join(snap.archiveSubPath(), snap.archiveName()) +} + // Ordering of elements in the CSI volume id. // ID is of the form {server}/{baseDir}/{subDir}. // TODO: This volume id format limits baseDir and @@ -74,6 +105,19 @@ const ( totalIDElements // Always last ) +// Ordering of elements in the CSI snapshot id. +// ID is of the form {server}/{baseDir}/{snapName}/{srcVolumeName}. +// Adding a new element should always go at the end +// before totalSnapIDElements +const ( + idSnapServer = iota + idSnapBaseDir + idSnapUUID + idSnapArchivePath + idSnapArchiveName + totalIDSnapElements // Always last +) + // CreateVolume create a volume func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { name := req.GetName() @@ -263,11 +307,115 @@ func (cs *ControllerServer) ControllerGetCapabilities(ctx context.Context, req * } func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { - return nil, status.Error(codes.Unimplemented, "") + if len(req.GetName()) == 0 { + return nil, status.Error(codes.InvalidArgument, "CreateSnapshot name must be provided") + } + if len(req.GetSourceVolumeId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "CreateSnapshot source volume ID must be provided") + } + + srcVol, err := getNfsVolFromID(req.GetSourceVolumeId()) + if err != nil { + return nil, status.Errorf(codes.NotFound, "failed to create source volume: %v", err) + } + snapshot, err := newNFSSnapshot(req.GetName(), req.GetParameters(), srcVol) + if err != nil { + return nil, status.Errorf(codes.NotFound, "failed to create nfsSnapshot: %v", err) + } + snapVol := volumeFromSnapshot(snapshot) + if err = cs.internalMount(ctx, snapVol, nil, nil); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount snapshot nfs server: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, snapVol); err != nil { + klog.Warningf("failed to unmount snapshot nfs server: %v", err) + } + }() + snapInternalVolPath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, snapVol), snapshot.archiveSubPath()) + if err = os.MkdirAll(snapInternalVolPath, 0777); err != nil { + return nil, status.Errorf(codes.Internal, "failed to make subdirectory: %v", err) + } + if err := validateSnapshot(snapInternalVolPath, snapshot); err != nil { + return nil, err + } + + if err = cs.internalMount(ctx, srcVol, nil, nil); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, srcVol); err != nil { + klog.Warningf("failed to unmount src nfs server: %v", err) + } + }() + + srcPath := getInternalVolumePath(cs.Driver.workingMountDir, srcVol) + dstPath := filepath.Join(snapInternalVolPath, snapshot.archiveName()) + klog.V(2).Infof("archiving %v -> %v", srcPath, dstPath) + out, err := exec.Command("tar", "-C", srcPath, "-czvf", dstPath, ".").CombinedOutput() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to create archive for snapshot: %v: %v", err, string(out)) + } + klog.V(2).Infof("archived %s -> %s", srcPath, dstPath) + + var snapshotSize int64 + fi, err := os.Stat(dstPath) + if err != nil { + klog.Warningf("failed to determine snapshot size: %v", err) + } else { + snapshotSize = fi.Size() + } + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: snapshot.id, + SourceVolumeId: srcVol.id, + SizeBytes: snapshotSize, + CreationTime: timestamppb.Now(), + ReadyToUse: true, + }, + }, nil } func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { - return nil, status.Error(codes.Unimplemented, "") + if len(req.GetSnapshotId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "Snapshot ID is required for deletion") + } + snap, err := getNfsSnapFromID(req.GetSnapshotId()) + if err != nil { + // An invalid ID should be treated as doesn't exist + klog.Warningf("failed to get nfs snapshot for id %v deletion: %v", req.GetSnapshotId(), err) + return &csi.DeleteSnapshotResponse{}, nil + } + + var volCap *csi.VolumeCapability + mountOptions := getMountOptions(req.GetSecrets()) + if mountOptions != "" { + klog.V(2).Infof("DeleteSnapshot: found mountOptions(%s) for snapshot(%s)", mountOptions, req.GetSnapshotId()) + volCap = &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + MountFlags: []string{mountOptions}, + }, + }, + } + } + vol := volumeFromSnapshot(snap) + if err = cs.internalMount(ctx, vol, nil, volCap); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount nfs server for snapshot deletion: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, vol); err != nil { + klog.Warningf("failed to unmount nfs server after snapshot deletion: %v", err) + } + }() + + // delete snapshot archive + internalVolumePath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, vol), snap.archiveSubPath()) + klog.V(2).Infof("Removing snapshot archive at %v", internalVolumePath) + if err = os.RemoveAll(internalVolumePath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to delete subdirectory: %v", err.Error()) + } + + return &csi.DeleteSnapshotResponse{}, nil } func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { @@ -325,6 +473,47 @@ func (cs *ControllerServer) internalUnmount(ctx context.Context, vol *nfsVolume) return err } +func (cs *ControllerServer) copyFromSnapshot(ctx context.Context, req *csi.CreateVolumeRequest, dstVol *nfsVolume) error { + snap, err := getNfsSnapFromID(req.VolumeContentSource.GetSnapshot().GetSnapshotId()) + if err != nil { + return status.Error(codes.NotFound, err.Error()) + } + snapVol := volumeFromSnapshot(snap) + + var volCap *csi.VolumeCapability + if len(req.GetVolumeCapabilities()) > 0 { + volCap = req.GetVolumeCapabilities()[0] + } + + if err = cs.internalMount(ctx, snapVol, nil, volCap); err != nil { + return status.Errorf(codes.Internal, "failed to mount src nfs server for snapshot volume copy: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, snapVol); err != nil { + klog.Warningf("failed to unmount src nfs server after snapshot volume copy: %v", err) + } + }() + if err = cs.internalMount(ctx, dstVol, nil, volCap); err != nil { + return status.Errorf(codes.Internal, "failed to mount dst nfs server for snapshot volume copy: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, dstVol); err != nil { + klog.Warningf("failed to unmount dst nfs server after snapshot volume copy: %v", err) + } + }() + + // untar snapshot archive to dst path + snapPath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, snapVol), snap.archivePath()) + dstPath := getInternalVolumePath(cs.Driver.workingMountDir, dstVol) + klog.V(2).Infof("copy volume from snapshot %v -> %v", snapPath, dstPath) + out, err := exec.Command("tar", "-xzvf", snapPath, "-C", dstPath).CombinedOutput() + if err != nil { + return status.Errorf(codes.Internal, "failed to copy volume for snapshot: %v: %v", err, string(out)) + } + klog.V(2).Infof("volume copied from snapshot %v -> %v", snapPath, dstPath) + return nil +} + func (cs *ControllerServer) copyFromVolume(ctx context.Context, req *csi.CreateVolumeRequest, dstVol *nfsVolume) error { srcVol, err := getNfsVolFromID(req.GetVolumeContentSource().GetVolume().GetVolumeId()) if err != nil { @@ -340,26 +529,26 @@ func (cs *ControllerServer) copyFromVolume(ctx context.Context, req *csi.CreateV volCap = req.GetVolumeCapabilities()[0] } if err = cs.internalMount(ctx, srcVol, nil, volCap); err != nil { - return status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err.Error()) + return status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err) } defer func() { if err = cs.internalUnmount(ctx, srcVol); err != nil { - klog.Warningf("failed to unmount nfs server: %v", err.Error()) + klog.Warningf("failed to unmount nfs server: %v", err) } }() if err = cs.internalMount(ctx, dstVol, nil, volCap); err != nil { - return status.Errorf(codes.Internal, "failed to mount dst nfs server: %v", err.Error()) + return status.Errorf(codes.Internal, "failed to mount dst nfs server: %v", err) } defer func() { if err = cs.internalUnmount(ctx, dstVol); err != nil { - klog.Warningf("failed to unmount dst nfs server: %v", err.Error()) + klog.Warningf("failed to unmount dst nfs server: %v", err) } }() // recursive 'cp' with '-a' to handle symlinks out, err := exec.Command("cp", "-a", srcPath, dstPath).CombinedOutput() if err != nil { - return status.Error(codes.Internal, fmt.Sprintf("%v: %v", err, string(out))) + return status.Errorf(codes.Internal, "failed to copy volume %v: %v", err, string(out)) } klog.V(2).Infof("copied %s -> %s", srcPath, dstPath) return nil @@ -369,7 +558,7 @@ func (cs *ControllerServer) copyVolume(ctx context.Context, req *csi.CreateVolum vs := req.VolumeContentSource switch vs.Type.(type) { case *csi.VolumeContentSource_Snapshot: - return status.Error(codes.Unimplemented, "Currently only volume copy from another volume is supported") + return cs.copyFromSnapshot(ctx, req, vol) case *csi.VolumeContentSource_Volume: return cs.copyFromVolume(ctx, req, vol) default: @@ -377,6 +566,40 @@ func (cs *ControllerServer) copyVolume(ctx context.Context, req *csi.CreateVolum } } +// newNFSSnapshot Convert VolumeSnapshot parameters to a nfsSnapshot +func newNFSSnapshot(name string, params map[string]string, vol *nfsVolume) (*nfsSnapshot, error) { + server := vol.server + baseDir := vol.baseDir + for k, v := range params { + switch strings.ToLower(k) { + case paramServer: + server = v + case paramShare: + baseDir = v + } + } + + if server == "" { + return nil, fmt.Errorf("%v is a required parameter", paramServer) + } + snapshot := &nfsSnapshot{ + server: server, + baseDir: baseDir, + uuid: name, + } + if vol.subDir != "" { + snapshot.src = vol.subDir + } + if vol.uuid != "" { + snapshot.src = vol.uuid + } + if snapshot.src == "" { + return nil, fmt.Errorf("missing required source volume name") + } + snapshot.id = getSnapshotIDFromNfsSnapshot(snapshot) + return snapshot, nil +} + // newNFSVolume Convert VolumeCreate parameters to an nfsVolume func newNFSVolume(name string, size int64, params map[string]string, defaultOnDeletePolicy string) (*nfsVolume, error) { var server, baseDir, subDir, onDelete string @@ -470,6 +693,17 @@ func getVolumeIDFromNfsVol(vol *nfsVolume) string { return strings.Join(idElements, separator) } +// Given a nfsSnapshot, return a CSI snapshot id. +func getSnapshotIDFromNfsSnapshot(snap *nfsSnapshot) string { + idElements := make([]string, totalIDSnapElements) + idElements[idSnapServer] = strings.Trim(snap.server, "/") + idElements[idSnapBaseDir] = strings.Trim(snap.baseDir, "/") + idElements[idSnapUUID] = snap.uuid + idElements[idSnapArchivePath] = snap.uuid + idElements[idSnapArchiveName] = snap.src + return strings.Join(idElements, separator) +} + // Given a CSI volume id, return a nfsVolume // sample volume Id: // @@ -513,6 +747,25 @@ func getNfsVolFromID(id string) (*nfsVolume, error) { }, nil } +// Given a CSI snapshot ID, return a nfsSnapshot +// sample snapshot ID: +// +// nfs-server.default.svc.cluster.local#share#snapshot-016f784f-56f4-44d1-9041-5f59e82dbce1#snapshot-016f784f-56f4-44d1-9041-5f59e82dbce1#pvc-4bcbf944-b6f7-4bd0-b50f-3c3dd00efc64 +func getNfsSnapFromID(id string) (*nfsSnapshot, error) { + segments := strings.Split(id, separator) + if len(segments) == totalIDSnapElements { + return &nfsSnapshot{ + id: id, + server: segments[idSnapServer], + baseDir: segments[idSnapBaseDir], + src: segments[idSnapArchiveName], + uuid: segments[idSnapUUID], + }, nil + } + + return &nfsSnapshot{}, fmt.Errorf("failed to create nfsSnapshot from snapshot ID") +} + // isValidVolumeCapabilities validates the given VolumeCapability array is valid func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error { if len(volCaps) == 0 { @@ -525,3 +778,32 @@ func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error { } return nil } + +// Validate snapshot after internal mount +func validateSnapshot(snapInternalVolPath string, snap *nfsSnapshot) error { + return filepath.WalkDir(snapInternalVolPath, func(path string, d fs.DirEntry, err error) error { + if path == snapInternalVolPath { + // skip root + return nil + } + if err != nil { + return err + } + if d.Name() != snap.archiveName() { + // there should be just one archive in the snapshot path and archive name should match + return status.Errorf(codes.AlreadyExists, "snapshot with the same name but different source volume ID already exists: found %q, desired %q", d.Name(), snap.archiveName()) + } + return nil + }) +} + +// Volume for snapshot internal mount/unmount +func volumeFromSnapshot(snap *nfsSnapshot) *nfsVolume { + return &nfsVolume{ + id: snap.id, + server: snap.server, + baseDir: snap.baseDir, + subDir: snap.baseDir, + uuid: snap.uuid, + } +} diff --git a/pkg/nfs/controllerserver_test.go b/pkg/nfs/controllerserver_test.go index 08663510e..8e5d43331 100644 --- a/pkg/nfs/controllerserver_test.go +++ b/pkg/nfs/controllerserver_test.go @@ -17,6 +17,8 @@ limitations under the License. package nfs import ( + "archive/tar" + "compress/gzip" "os" "path/filepath" "reflect" @@ -31,6 +33,7 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" mount "k8s.io/mount-utils" ) @@ -352,6 +355,13 @@ func TestControllerGetCapabilities(t *testing.T) { }, }, }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + }, + }, + }, }, }, expectedErr: nil, @@ -687,6 +697,8 @@ func TestCopyVolume(t *testing.T) { req *csi.CreateVolumeRequest dstVol *nfsVolume expectErr bool + prepare func() error + cleanup func() error }{ { desc: "copy volume from valid volume", @@ -707,6 +719,56 @@ func TestCopyVolume(t *testing.T) { subDir: "subdir", uuid: "dst-pv-name", }, + prepare: func() error { return os.MkdirAll("/tmp/src-pv-name/subdir", 0777) }, + cleanup: func() error { return os.RemoveAll("/tmp/src-pv-name") }, + }, + { + desc: "copy volume from valid snapshot", + req: &csi.CreateVolumeRequest{ + Name: "snapshot-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + }, + }, + }, + }, + dstVol: &nfsVolume{ + id: "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name", + server: "//nfs-server.default.svc.cluster.local", + baseDir: "share", + subDir: "subdir", + uuid: "dst-pv-name", + }, + prepare: func() error { + if err := os.MkdirAll("/tmp/snapshot-name/share/snapshot-name/", 0777); err != nil { + return err + } + file, err := os.Create("/tmp/snapshot-name/share/snapshot-name/src-pv-name.tar.gz") + if err != nil { + return err + } + defer file.Close() + gzipWriter := gzip.NewWriter(file) + defer gzipWriter.Close() + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + body := "test file" + hdr := &tar.Header{ + Name: "test.txt", + Mode: 0777, + Size: int64(len(body)), + } + if err := tarWriter.WriteHeader(hdr); err != nil { + return err + } + if _, err := tarWriter.Write([]byte(body)); err != nil { + return err + } + return nil + }, + cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") }, }, { desc: "copy volume missing source id", @@ -747,17 +809,263 @@ func TestCopyVolume(t *testing.T) { }, expectErr: true, }, - } - if err := os.MkdirAll("/tmp/src-pv-name/subdir", 0777); err != nil { - t.Fatalf("Unexpected error when creating srcVolume: %v", err) + { + desc: "copy volume from broken snapshot", + req: &csi.CreateVolumeRequest{ + Name: "snapshot-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + }, + }, + }, + }, + dstVol: &nfsVolume{ + id: "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name", + server: "//nfs-server.default.svc.cluster.local", + baseDir: "share", + subDir: "subdir", + uuid: "dst-pv-name", + }, + expectErr: true, + }, + { + desc: "copy volume from missing snapshot", + req: &csi.CreateVolumeRequest{ + Name: "snapshot-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{}, + }, + }, + }, + dstVol: &nfsVolume{ + id: "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name", + server: "//nfs-server.default.svc.cluster.local", + baseDir: "share", + subDir: "subdir", + uuid: "dst-pv-name", + }, + expectErr: true, + }, + { + desc: "copy volume from snapshot into missing dst volume", + req: &csi.CreateVolumeRequest{ + Name: "snapshot-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{}, + }, + }, + }, + dstVol: &nfsVolume{ + server: "//nfs-server.default.svc.cluster.local", + baseDir: "share", + subDir: "subdir", + uuid: "dst-pv-name", + }, + expectErr: true, + }, } for _, test := range cases { t.Run(test.desc, func(t *testing.T) { + if test.prepare != nil { + if err := test.prepare(); err != nil { + t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err) + } + } cs := initTestController(t) err := cs.copyVolume(context.TODO(), test.req, test.dstVol) if (err == nil) == test.expectErr { t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err) } + if test.cleanup != nil { + if err := test.cleanup(); err != nil { + t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err) + } + } }) } } + +func TestCreateSnapshot(t *testing.T) { + cases := []struct { + desc string + req *csi.CreateSnapshotRequest + expResp *csi.CreateSnapshotResponse + expectErr bool + prepare func() error + cleanup func() error + }{ + { + desc: "create snapshot with valid request", + req: &csi.CreateSnapshotRequest{ + SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name", + Name: "snapshot-name", + }, + expResp: &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name", + ReadyToUse: true, + SizeBytes: 1, // doesn't match exact size, just denotes non-zero size expected + CreationTime: timestamppb.Now(), // doesn't match exact timestamp, just denotes non-zero ts expected + }, + }, + prepare: func() error { return os.MkdirAll("/tmp/src-pv-name/subdir", 0777) }, + cleanup: func() error { return os.RemoveAll("/tmp/src-pv-name") }, + }, + { + desc: "create snapshot from nonexisting volume", + req: &csi.CreateSnapshotRequest{ + SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name", + Name: "snapshot-name", + }, + expectErr: true, + }, + } + for _, test := range cases { + t.Run(test.desc, func(t *testing.T) { + if test.prepare != nil { + if err := test.prepare(); err != nil { + t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err) + } + } + cs := initTestController(t) + resp, err := cs.CreateSnapshot(context.TODO(), test.req) + if (err == nil) == test.expectErr { + t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err) + } + if err := matchCreateSnapshotResponse(test.expResp, resp); err != nil { + t.Errorf("[test: %s] failed %q: got resp %+v, expected %+v", test.desc, err, resp, test.expResp) + } + if test.cleanup != nil { + if err := test.cleanup(); err != nil { + t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err) + } + } + }) + } +} + +func TestDeleteSnapshot(t *testing.T) { + cases := []struct { + desc string + req *csi.DeleteSnapshotRequest + expResp *csi.DeleteSnapshotResponse + expectErr bool + prepare func() error + cleanup func() error + }{ + { + desc: "delete valid snapshot", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + }, + expResp: &csi.DeleteSnapshotResponse{}, + prepare: func() error { + if err := os.MkdirAll("/tmp/snapshot-name/snapshot-name/", 0777); err != nil { + return err + } + f, err := os.OpenFile("/tmp/snapshot-name/snapshot-name/src-pv-name.tar.gz", os.O_CREATE, 0777) + if err != nil { + return err + } + return f.Close() + }, + cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") }, + }, + { + desc: "delete nonexisting snapshot", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + }, + expResp: &csi.DeleteSnapshotResponse{}, + }, + { + desc: "delete snapshot with improper id", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: "incorrect-snap-id", + }, + expResp: &csi.DeleteSnapshotResponse{}, + }, + { + desc: "delete valid snapshot with mount options", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + Secrets: map[string]string{"mountoptions": "nfsvers=4.1"}, + }, + expResp: &csi.DeleteSnapshotResponse{}, + prepare: func() error { + if err := os.MkdirAll("/tmp/snapshot-name/snapshot-name/", 0777); err != nil { + return err + } + f, err := os.OpenFile("/tmp/snapshot-name/snapshot-name/src-pv-name.tar.gz", os.O_CREATE, 0777) + if err != nil { + return err + } + return f.Close() + }, + cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") }, + }, + } + for _, test := range cases { + t.Run(test.desc, func(t *testing.T) { + if test.prepare != nil { + if err := test.prepare(); err != nil { + t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err) + } + } + cs := initTestController(t) + resp, err := cs.DeleteSnapshot(context.TODO(), test.req) + if (err == nil) == test.expectErr { + t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err) + } + if !reflect.DeepEqual(test.expResp, resp) { + t.Errorf("[test: %s] got resp %+v, expected %+v", test.desc, resp, test.expResp) + } + if test.cleanup != nil { + if err := test.cleanup(); err != nil { + t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err) + } + } + }) + } +} + +func matchCreateSnapshotResponse(e, r *csi.CreateSnapshotResponse) error { + if e == nil && r == nil { + return nil + } + if e == nil || e.Snapshot == nil { + return fmt.Errorf("expected nil response") + } + if r == nil || r.Snapshot == nil { + return fmt.Errorf("unexpected nil response") + } + es, rs := e.Snapshot, r.Snapshot + + var errs []string + // comparing ts and size just for presence, not the exact value + if es.CreationTime.IsValid() != rs.CreationTime.IsValid() { + errs = append(errs, "CreationTime") + } + if (es.SizeBytes == 0) != (rs.SizeBytes == 0) { + errs = append(errs, "SizeBytes") + } + // comparing remaining fields for exact match + if es.ReadyToUse != rs.ReadyToUse { + errs = append(errs, "ReadyToUse") + } + if es.SnapshotId != rs.SnapshotId { + errs = append(errs, "SnapshotId") + } + if es.SourceVolumeId != rs.SourceVolumeId { + errs = append(errs, "SourceVolumeId") + } + if len(errs) == 0 { + return nil + } + return fmt.Errorf("mismatch CreateSnapshotResponse in fields: %v", strings.Join(errs, ", ")) +} diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index 8230bbf28..e312f05c1 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -88,6 +88,7 @@ func NewDriver(options *DriverOptions) *Driver { csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, + csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, }) n.AddNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{ diff --git a/test/external-e2e/run.sh b/test/external-e2e/run.sh index de1264858..8790da267 100644 --- a/test/external-e2e/run.sh +++ b/test/external-e2e/run.sh @@ -32,8 +32,11 @@ setup_e2e_binaries() { # test on alternative driver name sed -i "s/nfs.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/storageclass-nfs.yaml + sed -i "s/nfs.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/snapshotclass-nfs.yaml # install csi driver - mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml + mkdir -p /tmp/csi + cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml + cp deploy/example/snapshotclass-nfs.yaml /tmp/csi/snapshotclass.yaml make e2e-bootstrap make install-nfs-server } diff --git a/test/external-e2e/testdriver.yaml b/test/external-e2e/testdriver.yaml index b63de8d90..f31623cd8 100644 --- a/test/external-e2e/testdriver.yaml +++ b/test/external-e2e/testdriver.yaml @@ -3,6 +3,8 @@ StorageClass: FromFile: /tmp/csi/storageclass.yaml +SnapshotClass: + FromName: true DriverInfo: Name: test.csi.k8s.io SupportedFsType: {"nfs"} @@ -13,6 +15,7 @@ DriverInfo: RWX: true fsGroup: true pvcDataSource: true + snapshotDataSource: true InlineVolumes: - Attributes: server: nfs-server.default.svc.cluster.local