From 1302b4369687bb0a03633efa153f51985f69a7b4 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Fri, 17 Mar 2023 17:12:01 +0100 Subject: [PATCH 1/5] feat: add csi-snapshotter sidecar and external-snapshotter $ helm package charts/latest/csi-driver-nfs -d charts/latest/ --- charts/latest/csi-driver-nfs-v0.0.0.tgz | Bin 3804 -> 10237 bytes .../templates/csi-nfs-controller.yaml | 16 + .../templates/csi-snapshot-controller.yaml | 51 ++ .../templates/rbac-csi-nfs.yaml | 11 + .../templates/rbac-snapshot-controller.yaml | 93 ++++ ....storage.k8s.io_volumesnapshotclasses.yaml | 146 ++++++ ...storage.k8s.io_volumesnapshotcontents.yaml | 473 ++++++++++++++++++ ...apshot.storage.k8s.io_volumesnapshots.yaml | 387 ++++++++++++++ charts/latest/csi-driver-nfs/values.yaml | 14 + deploy/csi-nfs-controller.yaml | 14 + deploy/csi-snapshot-controller.yaml | 40 ++ deploy/install-driver.sh | 13 + deploy/rbac-csi-nfs.yaml | 9 + deploy/rbac-snapshot-controller.yaml | 82 +++ ....storage.k8s.io_volumesnapshotclasses.yaml | 144 ++++++ ...storage.k8s.io_volumesnapshotcontents.yaml | 471 +++++++++++++++++ ...apshot.storage.k8s.io_volumesnapshots.yaml | 385 ++++++++++++++ deploy/uninstall-driver.sh | 6 + hack/verify-helm-chart.sh | 8 +- test/external-e2e/run.sh | 5 +- test/external-e2e/testdriver.yaml | 3 + 21 files changed, 2368 insertions(+), 3 deletions(-) create mode 100644 charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml create mode 100644 charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml create mode 100644 charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml create mode 100644 charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml create mode 100644 charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml create mode 100644 deploy/csi-snapshot-controller.yaml create mode 100644 deploy/rbac-snapshot-controller.yaml create mode 100644 deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml create mode 100644 deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml create mode 100644 deploy/snapshot.storage.k8s.io_volumesnapshots.yaml diff --git a/charts/latest/csi-driver-nfs-v0.0.0.tgz b/charts/latest/csi-driver-nfs-v0.0.0.tgz index 0ef63816cffa7a70cb03f8a3ef1ee222c33b42c8..ea01024fa47be57cb2f54a76a184d03a9f865c49 100644 GIT binary patch literal 10237 zcmZX4Ra6`RuO@BsMb>8&j_a(KRvBtFG;be|7LA?f8fI_kf5{y(TnMU|B3=)o8-qvuVdnTQ z*6{4rP486Np!>z39v(z^x6WB!FGGygy&mg3N4KV zlg%MJpbcc;{lzh=;NrXOUNagUW}fmegS z^&nqw;K~!3J~5TZvCQsYwS1Aj+|_rWE_ejm2RyQVPV*9yderMQWm zSrtiK6_FmgLG?IRr&8ggefd62&{^G@Sr8!W*#Dot*0Yvn7 z?q2#tN|;0Agz@g#!{qEG`RD(hX4F1MgZyOy(tYV2`E;Ep?4nS>`6YjSjjE1T+rPWz zFcn1&@0c6^K5jDz!!{( zpYmmWgi9jEP2q~psF4b@;e$Ozl|8^sj+&r|PSS#%Uwo;gVwUq-{QlDuM=KxB1W2Yf zo8oku*V94662HUghpZRQ6rjgb?eY<#X)*mgZv?Gul9QGm&9?GpB9*;{i`3nJ$;_S^ zlwe1SkYWo-@qd}{1hB8x3qSgx)4NQZQk~ccMM(z@5|ne5^Q!jQd$-CP$4>R4>T$`P z=)EF*88pL#ecMY4_P{ei8DyLu%Y%*Ie~)zrO8Exz2`KB6)~_bi1zE?U`yd%`1V|pA z#{n`4Gh$On#VBwI4dP9r=cNHn;ny%J(n*L}Xaa$)aq&g!Ca>Ri#Xi$UUW_sm%apwf zp>n=%&6!DZjqo8qbiey&dag%?4(P7zoun!QGJGi6`u@QHkf`<^X-5G&+NZ6{sRX|m zwc}Nc#$l%b1cnE%#kvLcrLNmG!1Uj>xJ{pDN)pmZok1r!FhHpfH4n@Q3VF)oTKH{t zk)qP`EnmWo3S#t-5&HLjc4-@va`)B|O3Y1Jfi?OtQr2yek+`Zr_X+!*i(YsM zsYh~-M%0h}%A=OuN(APCbA1zw;d?vcgUN~x+K1-m8K8VHESkv0Tx`f%zS$5o@*hzd z6CThXGq7EWIURNQ4B?zlr!qOO{IE5P1z?Du-vWr+AaX9vR1=1!qsA@=M8(3hC1ES< z@x(D>{=di&1s2!11zj6^h;Dm-1JgqakPrREZo1~i8$ zN8vgc!dWU2Kl)^#!n-2XR2b1R;Ydp}sny3NBh^8bd`s{Z3`=c7HV;F9rACtof%STM z;ULNnb>tD443ISHq9Z}A>lInKxUe9jm62=#s7uQW2F+xA_va+gHzS-N%b75`nE(4J zJVhZJ6jLbgc!r{1|LOUZuk`_mc=K)Fd<~2|x;x4P&4dh?QHd1MM~Sdtdl51=z_}8D zK^|RxCl79o%LQ2S;#5b!cw7fcU#~=lW8K=P3lPT=sc@8y2q6_ZWN6Vmx_PA8bSZ}5 ze$S!3D%~yvs3R)4`mc-kGhrFOB*n6Q?9;C(#vibO(ag91 zEV7A>4?!gkT;u6%r*FkEPdNl?0-;W7&W4DLq=^$sM4a7gf*A5_;TcR3}b>eSsw-A&3UgMc| z04E99ton3N#9|r^WseGzKW5Vq6)QsfUmvIlPO7=*3r(R^?uw>v7KegAUw$%f7=j9s zCfwZ&F1AtEy#&nKlYVCZ=T_qrLa4k0XQ^8iPA{3OO)Y6IRpoFJFl6$FYJ2F%_FopW zJBQ|cCIlebGQm?}a2<3Jl7+ddlBBHG#Vw4_r0J9N_1ZEpJCF$q7_#xs%9CGbH&koC z59>?1NC4tTk--ab-f#)v+eeH395>0FXIM@*;;(C{KF%I# zt*xAZ*1At^-`8K$h*YOk8giTjCFG->zSca8xs80wiZ;D#|||&$y{r!kVt~$cP~ka=m8% z`+yje;rp?c02|JXE{-nN@%}NdH9UM&`<*4tt`V7W4`mcL*!eIyj{iO5b>s?lAk88WD7q*{H;`lV!W=ieK&gza6wi@jpU-9~^FWviHG+sMj(} zQTruZpuFyu8_Ky3{1Q=FX5+me0fJ_X@kSfVVq@-5i%zaK&2!m%;4PZ~ zDBN>1b1lKjm>QS+z349xWkUTUH_`99XH>Mni3u|E)g242emKkp7tkvuSm|we7RQ`668&@lt5q=-Wa0oT$w6?URzy zHfi^#m&-Z!hDd)HCWF!E=gI8g=l5+?6C-XiVG-7srJ_flV_Rl_X8t z(V1;S8d4Fb$7&;_-%x;_ z)*RfAVT|1ac(~Y~rm^`6vJU6z8QojV-HJijc(1~SotsolID^sFTQ0de3@#Y68Q=L) zf}S|0uml?C>LAR>$UHuT@fG7ILlprBd+r&HEwF^NEj(1m>YVnamNWu~pgp*WEY>}S z>BpaqCb?e2+0PTBG9V2ea>aesBh!GI$|QHOv~1z#oR%A%69~_DVN z31~UQoQq-opur2royXZFDv>lnG`R-Qi%IzDviJyCG{m!F-Idb67kP7m(82lu@TG>r{&umMM#8&_K}B_1iG$*jkLZ|eS>;g*A9nMz#*P}r-$q(v z=U5xqVehp?Z3ulq}el_4G5 zm)#OFbqj^<8CTajm)>-(;nJ9AZgcB<;a8!1U)Y>YtTt&Otv{2(IX_K&K6cAMMu7`8 z)AL#&L*dLjS2Rew6}7-`bp|081y_EgmJ*}9h?)_B{i;eqJ=$>EE@%l?~o!gczI$RpBEyVord)SQu8e8X(q4u zPhE1H11H83mQTNM(IY2ZMEVw&Eke%dRl1`Xy(Hg{UUm&jK0%56tEr7da5SLyjfQpAKtQRulVihetAnPazF--O_^R^rqDYS zFob2W;OXY9>c^u{j=Lhk64A-*2EZjH$_`gv&xD%2aG7 zh5_U;6|ttU6jPlx7~WTi!h;5p!9;^3dSX@xRVq=8nT1yb(=#f}wNbfG%`^P+GB_rP zX$}`iB--R`T`<9L$|ihMr_5Xo{ac&r=g%~+0Rqn>dp(Y4oriVzT=5&W;)~$Tg1ee0 ztyWQgWwsLeHba%8h?_RSKj{oFKAr8iC66+jwz`?e*|W5&)p-PWjEzpIlP&7WqW+C5 zl0S}yVNB5Jm_ulZXbP>*jGK%?1f5L&Ge2BMNr#nT@%H7Ga7ujti8ywtp<8|$E{(?2 zgOWsFuQ8<49YRq)5YJ7QTOu@X8>>r^Spk_XJIC;;ar)k#F`yd^j!9+w;<11syXJt5 z%N6c}TKt>SAxBXk*wt9{+XnZyhv$b>kh01BG85wVGNT_^`+2CFP>r~O8XF?@d{P?QW0xq|;j$WtK=_XfF znW)S$P7P?4`(BlXPn%PzLC=K7nWF1RSj2Lw@2e~j7kgwDu$xbq9KZoCpugvt z;2I+3TNCB60_{*Wllsxjn1{NlV=Z((3<;SI;^{wM538XNVs8?awQ62oXBwUZA`cDb zX#~fXw-JX;JtZDn!hVKLU5dMC*$yLOuP?3vJx}(pFr*pVI(;p7r8mtzjX=}tzOixg zi#%;PSGsfsgLq<)Lq&4iWysYKbcYy3*z4w>_afxYw3Z)oBbp!g(S5Yt#m8Q=J;8Hw zLx<+8m)H1IliSJDXhJ%;e71dhX`Nt4I22Tt3ImB|O8L<&wzNzMDi_&Ws(99wk2GDV zj6=4w7W%2<{PZmVcSwES>CC^T+vuAW7FpkWHP*ygwn2_IeZ0&Q)L1|z`Nn`~V^UGK zsP&Fe(>qr&#D}PhL;PzKFzZ+~%uTR4h zUoyMmp*J3r*R|gu^DXYRK(vN;uqQEy2SsZkko3Hpw`wIYqWx`}|Luw*AMK*MLNElo zNT`rcr`3?NX4+q<<@S)kZ8O(Y)3ZFog{k4W>49i6=(M<7Mz#pU>!j&ykgmHKo!0zU zbt`<>0VQorjH0E))ch!`alPx&C^?2XttTPSYUIRD(F6VK*ISK~qqmFnsAECgJ@4(_+k^kQ%Q zkJWucx=xl_1dT*ltp(ChI8tmB7=vlisbEU|k}Q>mJI2fX+(tL<@3e_iy+3S*pNzT1 zeJ9bAU=-Xe)R`zj z2`3yW^CtA^$p~od6fPcM53`CHbjZdu#}nVm>ni)!`gG_0T6X(r^Gg`E!Mf{=iw>roz!y55qE~c|UtuSk)9WE%BaXn7-7T}&&36!jj@e7aYUM0LV}wm~uE*|!0r;=e z9`@!+Bbkqx{J0Ud3?z-zNACXBe}cQP$AVCecl$_Q4xGU z`d!PiFosixAmc^f8sW0w?2PQq_R^O8?a;dAv(6e1w$wwz*Qq&K=O$R!eAJKrzHmaL zr!_EBZ@<@=?_(N&KKzV>r{t-`dber_6LFL|a5tN+-r3^@eI zeH0@&eZKx&iYP4-ZB-7FqMoSN7EmLD0Z*@e)hkhg@TINsN81&; z&{~Hpt2Yl4M!hb^mcXGXna!FytLBL1HaC$IZN*aL{jDL2w%kxCt>}6<2<2i116~r5 zFf1{Gi{Z_?cIA`BO_u}Phq79kENhr)w~Kr2H=CChw|`5;{wJIl+tHv%gD zM9CuM>fB0(Jfm*y@1`sc0Y|qp7}UEM#t|%f$BU~`jLr5Ho$!McPz_LFvs{PjXI%^D zxVqL1JIG)db|*cc|xn=_>z^iaXA;XezZx{e~Z_KmYsQY>}J&go>{N z2QE<>NvSe8omNQ}6Z>O2NQJ}oaLs3)IP9w7_~+|Qpw`c8hrMoYgK{pY4N{1pz{Y0v z7R39xqY~nqSr!A?V*1Rv`RGIp`bOQt24}xkg#x3Osu0wI1;2#3qtUxflE#k=Nq_vC zvQc1?K!OpU?l1*Nl&}2K_l(GEJsay}L(7iD^Ve@8pLJn|WmO5r_A}FUo3Zh`0znMa zJk9&GYsqM7({PlkY7)6HKA*V|RWyy9PJ@@f^(nMrVu8Qc>lUE|ZmU&1Bk0exVx=1o zE4RLaQC#iniuS=MW3WSW1JQAsh|}_A2iB7$B%>dZGQ*XZ)&@nTVhLAmbj zsH44FFn594?!~yi@J(=U1UX}uq2uCq)Od5fhxJ1VVN&=*f61sa)w6-<|Kbrg(6^kO z#v+t;UYhh`S~t%LgV*K33R&}?_|FQuDL}n=mgp9J!j7W4Ia>ZOsK_maYIf+4*cX1f z!%;YNU;nSS@?=C+oC{^siKfcmmDQr+O1>GqnK#X?*dH?5*mH13vj$RBk9Q@O#kPzu zT3s@*RBI1m#>ak_bZuotS>9Ho;85}xj4vT=@#!oL&b$S1$ozbX5}}G$E5BNvixBcD zw6qQ2cV2In+GiT=J~qznR!CbaQ6kBH<7`W5qB;qF!^+K^;S8qE4QN;Uc+kR+ru5>a)* zRCHStv9_6^aKQ~VM5uV3b%uzp4k_MOO*9CJ8ol&TyLPrFabKN7f9~4exT(C5ucMki zv}~fD`{_?-rLZ1s--Tx(JDC0k!W3C-c7MJY4L$%MuSbE5Q4h6&pUW+nUur%drXi$7 zDfFs%GDFZ2DpX+nLsji0ywoWuzCFCcVf_2G}6X2 zY%#6p>s$?$+kf7O8OT`G9ccrD^z-=&x!<;OK(t{~%{hEekA+ogKTfPjoJX+Sn||Ua zpWyuEmJ%)aXL|qQ0J@F!sS^R99j>qHwlp&!qR|9lP|J^qIzuTmZ*Mrp6J5&RAOVkJ zqKofdZ{PXmZ6St}Wm-1e9n$0kKEl-or^$jpJI%m^+CGQng@(#X_G{azFP^^sGzq@T z2n}V)tAD3pehlbRq#3xYO?t1B7pW7s>RgrS)gLVRx?-Hv>hs=Q7J2SlO7g8Nhpo|~$vW0N^4khs+FPbXQ2 zQsqTQOq`P+-#WieGK8DjK8Nj5g42Xu^;tu88Yaat68>o9T&Y~&C=XMhs6j>Fx?GdpvsSZ)o(oY$oY%oA#wFUVbOc6z&wZaJWO%zX@yN8MC%~_q#TIfs`Y5T4|L=L zxIT}r^)p-~Jps8F6INPpW*SyTX!y5NVyb=>-nvV+j8Qx9u~Whf@^S)5<_|g>TLAWB zxQgTF?~~A_g|F+mg!oJ#`O(y>*1AAGU;sg>(rBEPb4*evJFNSbD zw?i@5Miq@T=OisVRe;;@+lA)X{2(_HPOHgX8ea4|FUO$;9*F1C?xMybYz(^a72*6* zTHJD>788NLFiWfSoB2;Fl)50FOp32esi8>KtRKPQRp12M%ntzpyKf_7>5njr=!~vu zhC|%`oczN4bMY3nY$?0-7qvtptqi(^yVcH<-|U64C^+oHLmf*qE$R1ZmS`}N#NGZa zTa9&Pwrx`v2-~zyIRE8b$@3xHmq8BCfd7yfrIlc_SS=AHqL~Q_KfKmDU*1DXi+B^i zPq#%{*YPl{YN~{w*2CRL1Y$!xUJKI04%}Bq9>UU{FePlR+XH>rh+;hu!3LZ}aSxEf z0+s|nk6^;172{>P*@|+%RoL9dMHAvp2EBDmys-@~?bu?WxzOI5V z>m#2CUC6GT4umxq!_6heDW@}h+UR8|t5MoU1)&|nrZZz1*Qmo{4`|$3!uVx>NlYxJ z82zbS@;S*HsVhityZlu%Bd4N%v%j7nC}DRC@<3J6x@m%$toc`7>2=DG&;3nY>t;bi zz@0W+a7M<&Bgbh$?`Wb9Z5oBq*-oouNu%6d!`2`z+ta0u;ZKchb*1WA#lmH~Y&E0W z+1SFG^+sB`yM?Vm)*7fvzPeK7)Hbz^sH8?}bQz=t-uG`KqLz#Jk2l!%-;^WkK#f%S zf5YtZPm-`;Pf<8^Vu}j94|FT?{LKW6Yub-1wdh30aN!rvQ60@Ah#2IV! z{1Jz&JUOa}wx>i+K=ij?LLoCzq+~Y}jFhsXmux+CFXZc~PATJ?wHMsY~xQ z$I9AL$iCSs5fs{54~jU}i%C&nZ{!t^#8M>CQl8Jg;*AMhkx$Z`S^x%~nLlY9zDkW> zxkyIVF#+QuF1hvV4}qBDcCO(DFf)*@z=8UGS{yG z=nal4kJ>YFGuD6SuB~7+xE+;NDqPsAF(SGBDSuMVc`a7YwXJw?2QZ)Oe#5HqVLQSuxQq)!XE6bMYV%4Rh0Tro$Nx_Q;CL^hB&F>_n9d{`?S zbb*^Lq_9n^SsnJ4(_UdJ)wqYi8aW(fRZ`~xE$$C|PxBF}r;UOWdSUk$L*FF!vJfSA{#9)V9X2BzDsA$uP zTKU7GZx`GQWsEo`*F#t%gvvY7;%r=LT;lh9iu>&I`woK0|2aB2$gM<8*O<(9hw71%O`3roCY>Nm;&BSDS@mwjM((0H7NRIH;^~O^>0ojeyjoZ7rooZ?;j9{+a8kZw#4u%@I$f zE2Jcp#V)y(stX!1k-terH91C%O`#eqk{raJ#;L=A!3y!+pPFN)rfs6gP%G#a7sM(zr6vJtXha+3%?zD#tBw^1dcsR#w!uite}s#PM&%&@?Rp!?rCUd6 zio+atcUFI#c z&zlIV>3ku7NPo`N_i4jG^63C?rTOQv_p^+$tKEkPu3!P*$D?iE#1~GdTe~!i(dV9# z&^h20Fs{8_zoAUdiaCX zt#Xo=s!5w=|83gU&F|bZqBlwa+nWlw28W=GmvxIzkF4&gd5h1}vhnu|_Q3XuZI-*1 z{|${6{+r(VA4&_g`u|Vaz6`ss-7C+92)LPnrE^wDb_e_EVqu=Und->gRn{TcL^g+rc1#GbGj20(KAe@*f{@r=Xi)aBLP$~ zN5GA@-tm{3fWD|?68hiU^qI&CS&XehytySnHcbo8NQ{ea3v>XSpPjomgTr6`)cNQg zM-H-qk2h!GQIPRFcVkDseRLyyY&qR>Y|lb+K3EGbenUn->G9mkpg4PbHL`zCVn|%& z6#iB8*+!*(u6UYM9+J7rfo!jfU9r+K;U4cl|2am(o@zoOO2)X3w;5jD_U#Wx(njJp dS_fR^U!LS&-UvXj|MTew5g5@EgtCN&`d`>|8wUUY delta 3781 zcmV;$4m$DuPuv}lJAYhTbK^F$&NF{SA2|<`IYa8=OLAD{hl%aUx{|S7mNQ#hsgw&M zTN2hFzy(0risSR!w*Zh5Nl_QyH#2x}sy`3HVx6|p=|8}~2y>GfZJKf%Px6}Fl`)@kkoqtZR_YHKOkcXuuQ<}(c zI*%T!KDob0Au0WWQc=M}m~=gorn9e%XjIyZQALub+CDqzAH&I~J{($=fuWG_b2dQ9 zQKRxWNic+}N_mqR`A*OYI^Ju~_^p&n&Hg8ZWvHIm0j#nA-JNc?ZvVZ#H~W8)5@kw@ zmGNOFsD?0L(7Ltsx z&4(&w*@*I4XH7;SM1&JLojpr4#?Azz(KLkP;fc^^5*2c78$^npNim_ubD(VILgI)j zEvLcN56ZApX2Ga+Iww$NWU?J>2c69rVRV8Vl{%ARfPeSrqv)QR?lmrY!4I2q!9|SD z<(|92$OC0!2o?4&3sMVw?FGH{e0d7xghu!=ibTe>;U|(vG=`vMhOh;HA0xM)KS16V zh-<9inlc6h%)7*PCz*;YfIe!_0)~Vs1UfVi;hJ#!FjHv0OAyyQFyjfXrCet-j(8BRY@sr28e9{)`!eS)+i%TdxVjG@Q4{OY$Z$wG)l=&&ooV)wVWfr3A8El*5^dMsO2UULoQBY{)+YtS z;A$2$87ob!oLjlgiPZEq)LKV3RB2UI;BbhM4xx1-`q3EUjA3gd`ndw6X33s?6#r4f zNK_(|5$2L>-Uk>>s4neI2`1M2?Q}Yysee~^#eZd}%B`X_3!&Rd7P|F{UX9Lv*F7on?`xW(S z`!Ra8wFT!mM2S2?vx^FjX8TNpL{Me}(JJ3u&5jD!CI~P=1mz>J!KE@JBo0FuYk!@p zu-&fukyu2kt%uWuh*3N$#lhmI3+b5?NNBuzG=*N$-4*%KS`M!4k+uhyn zzMcPGqQJ!%>!=P@FhZ@3!N!;}F@MF_ShS72Bj@e|&$|#1jWN2a_fAEuKxv{g^#M|b zM4<^BL(c;+h8}4KzHbasr6j`r+gk|E(Y6atil=aQXLxYLd|&0Bf4@c2v=(+yt~B8d zgy8P31-^exbToc#G5^PwrS^Y0MwX&ffleO_1+B6FZg0De|L*N{ci!y(C4b88ZTme; zXcB@#4MWOMPgC4aj98*EhVZ_9cjtMw+0ji(cno@sb{{pTYV$d=IRo47_SQFNEXomD z=3H!=wZNa(*hn*QSq5vP!54=(nwF;%l#<2>f12Ykk6>@xexyl18xH9WwEQ_Srta;BzJ6 z&Yk3~Y%u4VHpO_xS5O|YEUr1`pzJ2@?mU&Eg=OdHW%KAoz2l$=8{OG9F@fMOGjDGS zNz?W7C@@mjNn50}xqnX?O?7<>d&)VO$#8>t?>P$hU=*MblPNVXT|W}fsV^Ls8$9?z z?-`pv%n6koxqC2nAknpuS9wwZwo0Hq5T*oh#MmMR7B>ezv|k($4KXdU6D7o|Sq`%X z^YN=0ij%t5R;x?2imm9H>T%Oh1vWJwX@*vnWmgl@f_N&J)_-iNtDCz}Au!6bn^}E* z_bn;^9J4Q}&T7y(gjTCU8hKpmyk@X+E?|QpD+uwDkypvREU;Rnu};>hq(V|XJzzvB zrwMARG$ww2qV*$5HH`>c$w3KooYSrVk@w(fhG&()H@}3}HYG4~bIl{TvULVktUQ>f z1>x>4T-REYQGXuY&0LNgmH2YJwfxC`ueI>dH{6mqmZ;SJx9>h49-bfd``cC;*A(+U>{0o$buIlS@ zR?3#H6Zt%|TK4n+6Px$5NwI|maJfaI$BvI0hmL#07k_Gh?m7X=Ah@-)dVpTuLGy++ zcHvLpxi)uuRfZKtnYAs!HQwkd&k{*|%GvZ>2>ppNbXGS@cvb~pVDo-8U%o==D`@^Y zov$|DOy#S}y=?JaOUt#dX@Vlt``c=vflGAwsgQcVc^9j)I8TgJCGMzA_IvlzzbfsA zsNV_&Gk;~dc0NG;D%`$^_VJaJzf8TFw-tY~ z%pUUGVY8n%0RfHozkPRddU$kseAu+*Adgc)xqsgO_T54M`10uF@a**X>u3I=}JJ(7S#qq?uBFAtI+3Q@KPcwTlzY#B#XJ7u?-p@Kd`SOGq|e zji?b`x?$$xZ`v?FNs-i57GOw`#pgI&tjX(~7cu7<8}*s}zH#L8k~Z(x8K*a4yS>bF z8-HKCppdi3%;lsX=cDY~8qOk17r>nF?Yye1eb3c!h{~ZAz9?M~T^Vj*lIp3wGj@Bo&|^_hmFBOF zx&0@VwfL{Aqs9%Ub(4(bps22U(Z@S$zNF``*s> zoBh8;Ddm3%CW80z0qGyBBhQ=o$$!$AXzW)~F($U5eDtfWDq@iO1=Kp5)%qg(aWTkw zncze6M>Rn_SN5ki%wHhyvjsxZ5#@y01WzP5ccC_1i;TrEKp04Ih1??VBGY9WpxkY= zC(Zy}Q{%9ukVtI_O6w$Rp{wlFtJM&jXaaX?LwtouOUA(Km z7Uyl~vZ#bBpd7rJJMN-HB1s637ojm9Hh71xtW^2Id)CS()XedG|J!$m=f_`;&M(i8 ze!A?R9{hZH_VMCx-!)Q6M2!2*q-r+(K(J_ct-)5U{P#sl4K1u+iFUWJ{kuD_o+z4ShWeE*4M4F`qLL9C#VVqt>aVptF?9Sq z92aQe)AL%w)oj0Em4VV)xvYE* z*DuMyOjKpv9=Nf9+en z=7ST9`Q&an*?tcgFk*xz8;|PJ5lv%N6R1ri=Pe>el-h8f8_o$Q$n^sxm(S({w9;k6 zt)Et;sIiSFozSBQMncG#^0EaqjbW|!d4$%u`Yw^uG~VytJbxQkGVUKKk0~ELhmF_X zGVllaP#b6GvcW$w(z%g0de?c#b)S7;Gw*7=QGSM9bcql5Z!lZM6@vaRc~`X-{{>@Y z2_11E@#)4`d;h0bkN~Y!GUE&&CD9cbp$fb$xENCf zDoZVB18R(n!GDN}flH24KKcL>8PRlt_V`wMPI&BX0Y`WAhIgsNA-%!aWhejVy8uo( zo7$ffN*zm(q69`c23~O3zwB!v(c6LpktBk{mxDgUR4OkRQQfxxJNCU`@P};M|BDCX zQQQ1id{qJSKj+TCqAk7{{Ou#_(CKVnS$fPBjp9D v6vpj&0gaKgT{|iM=>-!Ni5S~|VSn&e-pX6~ddj~800960h|-pD0A2t97)EfE diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml index 255d93fe2..1f5b0c773 100644 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml @@ -63,6 +63,22 @@ spec: resources: {{- toYaml .Values.controller.resources.csiProvisioner | nindent 12 }} securityContext: readOnlyRootFilesystem: true + {{- if .Values.externalSnapshotter.enabled }} + - name: csi-snapshotter + image: "{{ .Values.image.csiSnapshotter.repository }}:{{ .Values.image.csiSnapshotter.tag }}" + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election-namespace={{ .Release.Namespace }}" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiSnapshotter.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- end }} - name: liveness-probe image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" args: diff --git a/charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml b/charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml new file mode 100644 index 000000000..01ff7a60e --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/csi-snapshot-controller.yaml @@ -0,0 +1,51 @@ +{{- if .Values.externalSnapshotter.enabled -}} +# This YAML file shows how to deploy the snapshot controller + +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }} + namespace: {{ .Release.Namespace }} +{{ include "nfs.labels" . | indent 2 }} + app: {{ .Values.externalSnapshotter.name }} +{{- with .Values.externalSnapshotter.labels }} +{{ . | toYaml | indent 4 }} +{{- end }} +{{- with .Values.externalSnapshotter.annotations }} + annotations: +{{ . | toYaml | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.externalSnapshotter.controller.replicas }} + selector: + matchLabels: + app: {{ .Values.externalSnapshotter.name }} + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: {{ .Values.externalSnapshotter.name }} + spec: + serviceAccountName: {{ .Values.externalSnapshotter.name }} + containers: + - name: {{ .Values.externalSnapshotter.name }} + image: {{ .Values.image.externalSnapshotter.repository }}:{{ .Values.image.externalSnapshotter.tag }} + args: + - "--v=5" + - "--leader-election=true" + - "--leader-election-namespace={{ .Release.Namespace }}" + imagePullPolicy: {{ .Values.image.externalSnapshotter.pullPolicy }} +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml b/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml index bc94431d1..66c76ff72 100644 --- a/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml +++ b/charts/latest/csi-driver-nfs/templates/rbac-csi-nfs.yaml @@ -32,6 +32,17 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] + {{- if .Values.externalSnapshotter.enabled }} + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["get", "update", "patch"] + {{- end }} - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml b/charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml new file mode 100644 index 000000000..f4fb5181b --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/rbac-snapshot-controller.yaml @@ -0,0 +1,93 @@ +{{- if .Values.externalSnapshotter.enabled -}} +# RBAC file for the snapshot controller. +# +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.externalSnapshotter.name }} + namespace: {{ .Release.Namespace }} +{{ include "nfs.labels" . | indent 2 }} + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }}-runner +{{ include "nfs.labels" . | indent 2 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] +{{- if .Values.externalSnapshotter.enabledDistributedSnapshotting }} + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +{{- end }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }}-role +{{ include "nfs.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.externalSnapshotter.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.externalSnapshotter.name }}-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }}-leaderelection + namespace: {{ .Release.Namespace }} +{{ include "nfs.labels" . | indent 2 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.externalSnapshotter.name }}-leaderelection + namespace: {{ .Release.Namespace }} +{{ include "nfs.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.externalSnapshotter.name }} +roleRef: + kind: Role + name: {{ .Values.externalSnapshotter.name }}-leaderelection + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml new file mode 100644 index 000000000..a4464c626 --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -0,0 +1,146 @@ +{{- if .Values.externalSnapshotter.enabled -}} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: [vsclass, vsclasses] + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying + storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its + bound VolumeSnapshot is deleted. Supported values are "Retain" and + "Delete". "Retain" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are kept. "Delete" means that + the VolumeSnapshotContent and its physical snapshot on underlying + storage system are deleted. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: [deletionPolicy, driver] + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; + use snapshot.storage.k8s.io/v1 VolumeSnapshotClass + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying + storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its + bound VolumeSnapshot is deleted. Supported values are "Retain" and + "Delete". "Retain" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are kept. "Delete" means that + the VolumeSnapshotContent and its physical snapshot on underlying + storage system are deleted. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: [deletionPolicy, driver] + type: object + served: false + storage: false + subresources: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml new file mode 100644 index 000000000..0ff8596e0 --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -0,0 +1,473 @@ +{{- if .Values.externalSnapshotter.enabled -}} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: [vsc, vscs] + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot + on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported + values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are deleted. For dynamically + provisioned snapshots, this field will automatically be filled + in by the CSI snapshotter sidecar with the "DeletionPolicy" field + defined in the corresponding VolumeSnapshotClass. For pre-existing + snapshots, users MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the CSI driver used to create + the physical snapshot on the underlying storage system. This MUST + be the same as the name returned by the CSI GetPluginName() call + for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should + be) dynamically provisioned or already exists, and just requires + a Kubernetes object representation. This field is immutable after + creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" + of a pre-existing snapshot on the underlying storage system + for which a Kubernetes object representation was (or should + be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + oneOf: + - required: [snapshotHandle] + - required: [volumeHandle] + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and + as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for + the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after + creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: [deletionPolicy, driver, source, volumeSnapshotRef] + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the CSI + snapshotter sidecar with the "creation_time" value returned from + CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this + field will be filled with the "creation_time" value returned from + the CSI "ListSnapshots" gRPC call if the driver supports it. If + not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an + int64. On Unix, the command `date +%s%N` returns the current time + in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. When restoring a volume from this snapshot, the size + of the volume MUST NOT be smaller than the restoreSize if it is + specified, otherwise the restoration will fail. If not specified, + it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: [spec] + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot + on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is + deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported + values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are deleted. For dynamically + provisioned snapshots, this field will automatically be filled + in by the CSI snapshotter sidecar with the "DeletionPolicy" field + defined in the corresponding VolumeSnapshotClass. For pre-existing + snapshots, users MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the CSI driver used to create + the physical snapshot on the underlying storage system. This MUST + be the same as the name returned by the CSI GetPluginName() call + for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should + be) dynamically provisioned or already exists, and just requires + a Kubernetes object representation. This field is immutable after + creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" + of a pre-existing snapshot on the underlying storage system + for which a Kubernetes object representation was (or should + be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and + as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for + the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after + creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: [deletionPolicy, driver, source, volumeSnapshotRef] + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the CSI + snapshotter sidecar with the "creation_time" value returned from + CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this + field will be filled with the "creation_time" value returned from + the CSI "ListSnapshots" gRPC call if the driver supports it. If + not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an + int64. On Unix, the command `date +%s%N` returns the current time + in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. When restoring a volume from this snapshot, the size + of the volume MUST NOT be smaller than the restoreSize if it is + specified, otherwise the restoration will fail. If not specified, + it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: [spec] + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml new file mode 100644 index 000000000..9447d45d2 --- /dev/null +++ b/charts/latest/csi-driver-nfs/templates/snapshot.storage.k8s.io_volumesnapshots.yaml @@ -0,0 +1,387 @@ +{{- if .Values.externalSnapshotter.enabled -}} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: [vs] + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name + of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the + existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate + from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage + of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the + underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot + requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of + the PersistentVolumeClaim object representing the volume from + which a snapshot should be created. This PVC is assumed to + be in the same namespace as the VolumeSnapshot object. This + field should be set if the snapshot does not exists, and needs + to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of + a pre-existing VolumeSnapshotContent object representing an + existing volume snapshot. This field should be set if the + snapshot already exists and only needs a representation in + Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: [persistentVolumeClaimName] + - required: [volumeSnapshotContentName] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be + used. A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one + VolumeSnapshotClass exist for a given CSI Driver and more than + one have been marked as default, CreateSnapshot will fail and + generate an event. Empty string is not allowed for this field.' + type: string + required: [source] + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and + VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the + VolumeSnapshotContent object to which this VolumeSnapshot object + intends to bind to. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: To avoid possible security issues, consumers + must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot + and VolumeSnapshotContent point at each other) before using this + object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the snapshot + controller with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may + indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue + on waiting for the snapshot to be created based on the type of + error reported. The snapshot controller will keep retrying when + an error occurs during the snapshot creation. Upon success, this + error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be + used to restore a volume. In dynamic snapshot creation case, this + field will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller + with the "size_bytes" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than + the restoreSize if it is specified, otherwise the restoration + will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: [spec] + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name + of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the + existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate + from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage + of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the + underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; + use snapshot.storage.k8s.io/v1 VolumeSnapshot + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot + requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of + the PersistentVolumeClaim object representing the volume from + which a snapshot should be created. This PVC is assumed to + be in the same namespace as the VolumeSnapshot object. This + field should be set if the snapshot does not exists, and needs + to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of + a pre-existing VolumeSnapshotContent object representing an + existing volume snapshot. This field should be set if the + snapshot already exists and only needs a representation in + Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be + used. A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one + VolumeSnapshotClass exist for a given CSI Driver and more than + one have been marked as default, CreateSnapshot will fail and + generate an event. Empty string is not allowed for this field.' + type: string + required: [source] + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and + VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the + VolumeSnapshotContent object to which this VolumeSnapshot object + intends to bind to. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: To avoid possible security issues, consumers + must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot + and VolumeSnapshotContent point at each other) before using this + object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the snapshot + controller with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may + indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue + on waiting for the snapshot to be created based on the type of + error reported. The snapshot controller will keep retrying when + an error occurs during the snapshot creation. Upon success, this + error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be + used to restore a volume. In dynamic snapshot creation case, this + field will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller + with the "size_bytes" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than + the restoreSize if it is specified, otherwise the restoration + will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: [spec] + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] +{{- end -}} diff --git a/charts/latest/csi-driver-nfs/values.yaml b/charts/latest/csi-driver-nfs/values.yaml index 97da53e85..c621a666f 100755 --- a/charts/latest/csi-driver-nfs/values.yaml +++ b/charts/latest/csi-driver-nfs/values.yaml @@ -8,6 +8,10 @@ image: repository: registry.k8s.io/sig-storage/csi-provisioner tag: v3.3.0 pullPolicy: IfNotPresent + csiSnapshotter: + repository: registry.k8s.io/sig-storage/csi-snapshotter + tag: v6.2.1 + pullPolicy: IfNotPresent livenessProbe: repository: registry.k8s.io/sig-storage/livenessprobe tag: v2.8.0 @@ -16,6 +20,10 @@ image: repository: registry.k8s.io/sig-storage/csi-node-driver-registrar tag: v2.6.2 pullPolicy: IfNotPresent + externalSnapshotter: + repository: registry.k8s.io/sig-storage/snapshot-controller + tag: v6.1.0 + pullPolicy: IfNotPresent serviceAccount: create: true # When true, service accounts will be created for you. Set to false if you want to use your own. @@ -110,6 +118,12 @@ node: cpu: 10m memory: 20Mi +externalSnapshotter: + enabled: true + name: snapshot-controller + controller: + replicas: 1 + ## Reference to one or more secrets to be used when pulling images ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## diff --git a/deploy/csi-nfs-controller.yaml b/deploy/csi-nfs-controller.yaml index c35cc5862..05fdfb46f 100644 --- a/deploy/csi-nfs-controller.yaml +++ b/deploy/csi-nfs-controller.yaml @@ -51,6 +51,20 @@ spec: requests: cpu: 10m memory: 20Mi + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election-namespace=kube-system" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /csi - name: liveness-probe image: registry.k8s.io/sig-storage/livenessprobe:v2.8.0 args: diff --git a/deploy/csi-snapshot-controller.yaml b/deploy/csi-snapshot-controller.yaml new file mode 100644 index 000000000..cd5b86836 --- /dev/null +++ b/deploy/csi-snapshot-controller.yaml @@ -0,0 +1,40 @@ +# This YAML file shows how to deploy the snapshot controller + +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: snapshot-controller + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: snapshot-controller + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: snapshot-controller + spec: + serviceAccountName: snapshot-controller + containers: + - name: snapshot-controller + image: registry.k8s.io/sig-storage/snapshot-controller:v6.1.0 + args: + - "--v=5" + - "--leader-election=true" + imagePullPolicy: IfNotPresent diff --git a/deploy/install-driver.sh b/deploy/install-driver.sh index 2c4006bc1..28d2d2e9c 100755 --- a/deploy/install-driver.sh +++ b/deploy/install-driver.sh @@ -16,6 +16,13 @@ set -euo pipefail +# min version of this nfs plugin supporting CSI snapshots +snap_ver='4.3.0' + +# external-snapshotter to install for snapshots to work +snap_controller_repo='github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.2.1' +snap_crd_repo='github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.2.1' + ver="master" if [[ "$#" -gt 0 ]]; then ver="$1" @@ -33,6 +40,12 @@ if [ $ver != "master" ]; then repo="$repo/$ver" fi +# check for min supported version for snapshots or master to install external-snapshotter +if [[ "$ver" == master || "$(printf '%s\n' "$ver" "$snap_ver" | sort -V | head -n1)" = "$snap_ver" ]]; then + kubectl kustomize "$snap_crd_repo" | kubectl apply -f - + kubectl -n kube-system kustomize "$snap_controller_repo" | kubectl apply -f - +fi + echo "Installing NFS CSI driver, version: $ver ..." kubectl apply -f $repo/rbac-csi-nfs.yaml kubectl apply -f $repo/csi-nfs-driverinfo.yaml diff --git a/deploy/rbac-csi-nfs.yaml b/deploy/rbac-csi-nfs.yaml index 45c80e932..081d76e5d 100644 --- a/deploy/rbac-csi-nfs.yaml +++ b/deploy/rbac-csi-nfs.yaml @@ -26,6 +26,15 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["get", "update", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/deploy/rbac-snapshot-controller.yaml b/deploy/rbac-snapshot-controller.yaml new file mode 100644 index 000000000..2e6431bd3 --- /dev/null +++ b/deploy/rbac-snapshot-controller.yaml @@ -0,0 +1,82 @@ +# RBAC file for the snapshot controller. +# +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system +subjects: + - kind: ServiceAccount + name: snapshot-controller +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml b/deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml new file mode 100644 index 000000000..f656a4a42 --- /dev/null +++ b/deploy/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -0,0 +1,144 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: [vsclass, vsclasses] + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying + storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its + bound VolumeSnapshot is deleted. Supported values are "Retain" and + "Delete". "Retain" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are kept. "Delete" means that + the VolumeSnapshotContent and its physical snapshot on underlying + storage system are deleted. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: [deletionPolicy, driver] + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; + use snapshot.storage.k8s.io/v1 VolumeSnapshotClass + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying + storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its + bound VolumeSnapshot is deleted. Supported values are "Retain" and + "Delete". "Retain" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are kept. "Delete" means that + the VolumeSnapshotContent and its physical snapshot on underlying + storage system are deleted. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: [deletionPolicy, driver] + type: object + served: false + storage: false + subresources: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] diff --git a/deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml b/deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml new file mode 100644 index 000000000..381466c03 --- /dev/null +++ b/deploy/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -0,0 +1,471 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: [vsc, vscs] + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot + on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported + values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are deleted. For dynamically + provisioned snapshots, this field will automatically be filled + in by the CSI snapshotter sidecar with the "DeletionPolicy" field + defined in the corresponding VolumeSnapshotClass. For pre-existing + snapshots, users MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the CSI driver used to create + the physical snapshot on the underlying storage system. This MUST + be the same as the name returned by the CSI GetPluginName() call + for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should + be) dynamically provisioned or already exists, and just requires + a Kubernetes object representation. This field is immutable after + creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" + of a pre-existing snapshot on the underlying storage system + for which a Kubernetes object representation was (or should + be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + oneOf: + - required: [snapshotHandle] + - required: [volumeHandle] + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and + as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for + the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after + creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: [deletionPolicy, driver, source, volumeSnapshotRef] + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the CSI + snapshotter sidecar with the "creation_time" value returned from + CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this + field will be filled with the "creation_time" value returned from + the CSI "ListSnapshots" gRPC call if the driver supports it. If + not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an + int64. On Unix, the command `date +%s%N` returns the current time + in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. When restoring a volume from this snapshot, the size + of the volume MUST NOT be smaller than the restoreSize if it is + specified, otherwise the restoration will fail. If not specified, + it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: [spec] + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot + on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is + deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported + values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical + snapshot on underlying storage system are deleted. For dynamically + provisioned snapshots, this field will automatically be filled + in by the CSI snapshotter sidecar with the "DeletionPolicy" field + defined in the corresponding VolumeSnapshotClass. For pre-existing + snapshots, users MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: [Delete, Retain] + type: string + driver: + description: driver is the name of the CSI driver used to create + the physical snapshot on the underlying storage system. This MUST + be the same as the name returned by the CSI GetPluginName() call + for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should + be) dynamically provisioned or already exists, and just requires + a Kubernetes object representation. This field is immutable after + creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" + of a pre-existing snapshot on the underlying storage system + for which a Kubernetes object representation was (or should + be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and + as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for + the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after + creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: [deletionPolicy, driver, source, volumeSnapshotRef] + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the CSI + snapshotter sidecar with the "creation_time" value returned from + CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this + field will be filled with the "creation_time" value returned from + the CSI "ListSnapshots" gRPC call if the driver supports it. If + not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an + int64. On Unix, the command `date +%s%N` returns the current time + in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. When restoring a volume from this snapshot, the size + of the volume MUST NOT be smaller than the restoreSize if it is + specified, otherwise the restoration will fail. If not specified, + it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: [spec] + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] diff --git a/deploy/snapshot.storage.k8s.io_volumesnapshots.yaml b/deploy/snapshot.storage.k8s.io_volumesnapshots.yaml new file mode 100644 index 000000000..3d27cc054 --- /dev/null +++ b/deploy/snapshot.storage.k8s.io_volumesnapshots.yaml @@ -0,0 +1,385 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/665 + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: [vs] + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name + of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the + existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate + from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage + of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the + underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot + requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of + the PersistentVolumeClaim object representing the volume from + which a snapshot should be created. This PVC is assumed to + be in the same namespace as the VolumeSnapshot object. This + field should be set if the snapshot does not exists, and needs + to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of + a pre-existing VolumeSnapshotContent object representing an + existing volume snapshot. This field should be set if the + snapshot already exists and only needs a representation in + Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: [persistentVolumeClaimName] + - required: [volumeSnapshotContentName] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be + used. A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one + VolumeSnapshotClass exist for a given CSI Driver and more than + one have been marked as default, CreateSnapshot will fail and + generate an event. Empty string is not allowed for this field.' + type: string + required: [source] + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and + VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the + VolumeSnapshotContent object to which this VolumeSnapshot object + intends to bind to. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: To avoid possible security issues, consumers + must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot + and VolumeSnapshotContent point at each other) before using this + object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the snapshot + controller with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may + indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue + on waiting for the snapshot to be created based on the type of + error reported. The snapshot controller will keep retrying when + an error occurs during the snapshot creation. Upon success, this + error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be + used to restore a volume. In dynamic snapshot creation case, this + field will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller + with the "size_bytes" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than + the restoreSize if it is specified, otherwise the restoration + will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: [spec] + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a + volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name + of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the + existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate + from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage + of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the + underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; + use snapshot.storage.k8s.io/v1 VolumeSnapshot + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint the + client submits requests to. Cannot be updated. In CamelCase. More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot + requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of + the PersistentVolumeClaim object representing the volume from + which a snapshot should be created. This PVC is assumed to + be in the same namespace as the VolumeSnapshot object. This + field should be set if the snapshot does not exists, and needs + to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of + a pre-existing VolumeSnapshotContent object representing an + existing volume snapshot. This field should be set if the + snapshot already exists and only needs a representation in + Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be + used. A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one + VolumeSnapshotClass exist for a given CSI Driver and more than + one have been marked as default, CreateSnapshot will fail and + generate an event. Empty string is not allowed for this field.' + type: string + required: [source] + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and + VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the + VolumeSnapshotContent object to which this VolumeSnapshot object + intends to bind to. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: To avoid possible security issues, consumers + must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot + and VolumeSnapshotContent point at each other) before using this + object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic + snapshot creation case, this field will be filled in by the snapshot + controller with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may + indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue + on waiting for the snapshot to be created based on the type of + error reported. The snapshot controller will keep retrying when + an error occurs during the snapshot creation. Upon success, this + error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered + error during snapshot creation if specified. NOTE: message + may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be + used to restore a volume. In dynamic snapshot creation case, this + field will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver + supports it, otherwise, this field will be set to "True". If not + specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller + with the "size_bytes" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than + the restoreSize if it is specified, otherwise the restoration + will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: [spec] + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: '' + plural: '' + conditions: [] + storedVersions: [] diff --git a/deploy/uninstall-driver.sh b/deploy/uninstall-driver.sh index d563fc981..acd40f558 100755 --- a/deploy/uninstall-driver.sh +++ b/deploy/uninstall-driver.sh @@ -16,6 +16,10 @@ set -euo pipefail +# external-snapshotter to install for snapshots to work +snap_controller_repo='github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.2.1' +snap_crd_repo='github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.2.1' + ver="master" if [[ "$#" -gt 0 ]]; then ver="$1" @@ -34,6 +38,8 @@ if [ $ver != "master" ]; then fi echo "Uninstalling NFS driver, version: $ver ..." +# this keeps VolumeSnapshot CRDs untouched +kubectl -n kube-system kustomize "$snap_controller_repo" | kubectl delete --ignore-not-found -f - kubectl delete -f $repo/csi-nfs-controller.yaml --ignore-not-found kubectl delete -f $repo/csi-nfs-node.yaml --ignore-not-found kubectl delete -f $repo/csi-nfs-driverinfo.yaml --ignore-not-found diff --git a/hack/verify-helm-chart.sh b/hack/verify-helm-chart.sh index 174f06f77..642eec226 100755 --- a/hack/verify-helm-chart.sh +++ b/hack/verify-helm-chart.sh @@ -62,12 +62,16 @@ pip install yq --ignore-installed PyYAML # Extract images from csi-nfs-controller.yaml expected_csi_provisioner_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[0].image | head -n 1)" -expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)" -expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)" +expected_csi_snapshotter_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[1].image | head -n 1)" +expected_liveness_probe_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[2].image | head -n 1)" +expected_nfs_image="$(cat ${PKG_ROOT}/deploy/csi-nfs-controller.yaml | yq -r .spec.template.spec.containers[3].image | head -n 1)" csi_provisioner_image="$(get_image_from_helm_chart "csiProvisioner")" validate_image "${expected_csi_provisioner_image}" "${csi_provisioner_image}" +csi_snapshotter_image="$(get_image_from_helm_chart "csiSnapshotter")" +validate_image "${expected_csi_snapshotter_image}" "${csi_snapshotter_image}" + liveness_probe_image="$(get_image_from_helm_chart "livenessProbe")" validate_image "${expected_liveness_probe_image}" "${liveness_probe_image}" diff --git a/test/external-e2e/run.sh b/test/external-e2e/run.sh index de1264858..8790da267 100644 --- a/test/external-e2e/run.sh +++ b/test/external-e2e/run.sh @@ -32,8 +32,11 @@ setup_e2e_binaries() { # test on alternative driver name sed -i "s/nfs.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/storageclass-nfs.yaml + sed -i "s/nfs.csi.k8s.io/$DRIVER.csi.k8s.io/g" deploy/example/snapshotclass-nfs.yaml # install csi driver - mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml + mkdir -p /tmp/csi + cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml + cp deploy/example/snapshotclass-nfs.yaml /tmp/csi/snapshotclass.yaml make e2e-bootstrap make install-nfs-server } diff --git a/test/external-e2e/testdriver.yaml b/test/external-e2e/testdriver.yaml index b63de8d90..f31623cd8 100644 --- a/test/external-e2e/testdriver.yaml +++ b/test/external-e2e/testdriver.yaml @@ -3,6 +3,8 @@ StorageClass: FromFile: /tmp/csi/storageclass.yaml +SnapshotClass: + FromName: true DriverInfo: Name: test.csi.k8s.io SupportedFsType: {"nfs"} @@ -13,6 +15,7 @@ DriverInfo: RWX: true fsGroup: true pvcDataSource: true + snapshotDataSource: true InlineVolumes: - Attributes: server: nfs-server.default.svc.cluster.local From aeb0a8d410c6ca84522528c148cbfd07764f28cb Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Sat, 18 Mar 2023 21:40:45 +0100 Subject: [PATCH 2/5] feat: add SnapshotClass and VolumeSnapshot example --- deploy/example/snapshot-nfs-dynamic.yaml | 9 +++++++++ deploy/example/snapshotclass-nfs.yaml | 7 +++++++ 2 files changed, 16 insertions(+) create mode 100644 deploy/example/snapshot-nfs-dynamic.yaml create mode 100644 deploy/example/snapshotclass-nfs.yaml diff --git a/deploy/example/snapshot-nfs-dynamic.yaml b/deploy/example/snapshot-nfs-dynamic.yaml new file mode 100644 index 000000000..7795d0681 --- /dev/null +++ b/deploy/example/snapshot-nfs-dynamic.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: test-nfs-snapshot +spec: + volumeSnapshotClassName: csi-nfs-snapclass + source: + persistentVolumeClaimName: pvc-nfs-dynamic diff --git a/deploy/example/snapshotclass-nfs.yaml b/deploy/example/snapshotclass-nfs.yaml new file mode 100644 index 000000000..d6b8e3332 --- /dev/null +++ b/deploy/example/snapshotclass-nfs.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: csi-nfs-snapclass +driver: nfs.csi.k8s.io +deletionPolicy: Delete From ce66bf8a858ee29b2466c8200157f8d7a1acb730 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Fri, 17 Mar 2023 17:11:06 +0100 Subject: [PATCH 3/5] feat: implement volume snapshots --- pkg/nfs/controllerserver.go | 298 +++++++++++++++++++++++++++++++++++- pkg/nfs/nfs.go | 1 + 2 files changed, 291 insertions(+), 8 deletions(-) diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index eb8c52b01..a1911ac04 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -18,6 +18,7 @@ package nfs import ( "fmt" + "io/fs" "os" "os/exec" "path/filepath" @@ -29,6 +30,7 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/klog/v2" ) @@ -59,6 +61,35 @@ type nfsVolume struct { onDelete string } +// nfsSnapshot is an internal representation of a volume snapshot +// created by the provisioner. +type nfsSnapshot struct { + // Snapshot id. + id string + // Address of the NFS server. + // Matches paramServer. + server string + // Base directory of the NFS server to create snapshots under + // Matches paramShare. + baseDir string + // Snapshot name. + uuid string + // Source volume. + src string +} + +func (snap nfsSnapshot) archiveSubPath() string { + return snap.uuid +} + +func (snap nfsSnapshot) archiveName() string { + return fmt.Sprintf("%v.tar.gz", snap.src) +} + +func (snap nfsSnapshot) archivePath() string { + return filepath.Join(snap.archiveSubPath(), snap.archiveName()) +} + // Ordering of elements in the CSI volume id. // ID is of the form {server}/{baseDir}/{subDir}. // TODO: This volume id format limits baseDir and @@ -74,6 +105,19 @@ const ( totalIDElements // Always last ) +// Ordering of elements in the CSI snapshot id. +// ID is of the form {server}/{baseDir}/{snapName}/{srcVolumeName}. +// Adding a new element should always go at the end +// before totalSnapIDElements +const ( + idSnapServer = iota + idSnapBaseDir + idSnapUUID + idSnapArchivePath + idSnapArchiveName + totalIDSnapElements // Always last +) + // CreateVolume create a volume func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { name := req.GetName() @@ -263,11 +307,115 @@ func (cs *ControllerServer) ControllerGetCapabilities(ctx context.Context, req * } func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { - return nil, status.Error(codes.Unimplemented, "") + if len(req.GetName()) == 0 { + return nil, status.Error(codes.InvalidArgument, "CreateSnapshot name must be provided") + } + if len(req.GetSourceVolumeId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "CreateSnapshot source volume ID must be provided") + } + + srcVol, err := getNfsVolFromID(req.GetSourceVolumeId()) + if err != nil { + return nil, status.Errorf(codes.NotFound, "failed to create source volume: %v", err) + } + snapshot, err := newNFSSnapshot(req.GetName(), req.GetParameters(), srcVol) + if err != nil { + return nil, status.Errorf(codes.NotFound, "failed to create nfsSnapshot: %v", err) + } + snapVol := volumeFromSnapshot(snapshot) + if err = cs.internalMount(ctx, snapVol, nil, nil); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount snapshot nfs server: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, snapVol); err != nil { + klog.Warningf("failed to unmount snapshot nfs server: %v", err) + } + }() + snapInternalVolPath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, snapVol), snapshot.archiveSubPath()) + if err = os.MkdirAll(snapInternalVolPath, 0777); err != nil { + return nil, status.Errorf(codes.Internal, "failed to make subdirectory: %v", err) + } + if err := validateSnapshot(snapInternalVolPath, snapshot); err != nil { + return nil, err + } + + if err = cs.internalMount(ctx, srcVol, nil, nil); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, srcVol); err != nil { + klog.Warningf("failed to unmount src nfs server: %v", err) + } + }() + + srcPath := getInternalVolumePath(cs.Driver.workingMountDir, srcVol) + dstPath := filepath.Join(snapInternalVolPath, snapshot.archiveName()) + klog.V(2).Infof("archiving %v -> %v", srcPath, dstPath) + out, err := exec.Command("tar", "-C", srcPath, "-czvf", dstPath, ".").CombinedOutput() + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to create archive for snapshot: %v: %v", err, string(out)) + } + klog.V(2).Infof("archived %s -> %s", srcPath, dstPath) + + var snapshotSize int64 + fi, err := os.Stat(dstPath) + if err != nil { + klog.Warningf("failed to determine snapshot size: %v", err) + } else { + snapshotSize = fi.Size() + } + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: snapshot.id, + SourceVolumeId: srcVol.id, + SizeBytes: snapshotSize, + CreationTime: timestamppb.Now(), + ReadyToUse: true, + }, + }, nil } func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { - return nil, status.Error(codes.Unimplemented, "") + if len(req.GetSnapshotId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "Snapshot ID is required for deletion") + } + snap, err := getNfsSnapFromID(req.GetSnapshotId()) + if err != nil { + // An invalid ID should be treated as doesn't exist + klog.Warningf("failed to get nfs snapshot for id %v deletion: %v", req.GetSnapshotId(), err) + return &csi.DeleteSnapshotResponse{}, nil + } + + var volCap *csi.VolumeCapability + mountOptions := getMountOptions(req.GetSecrets()) + if mountOptions != "" { + klog.V(2).Infof("DeleteSnapshot: found mountOptions(%s) for snapshot(%s)", mountOptions, req.GetSnapshotId()) + volCap = &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + MountFlags: []string{mountOptions}, + }, + }, + } + } + vol := volumeFromSnapshot(snap) + if err = cs.internalMount(ctx, vol, nil, volCap); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount nfs server for snapshot deletion: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, vol); err != nil { + klog.Warningf("failed to unmount nfs server after snapshot deletion: %v", err) + } + }() + + // delete snapshot archive + internalVolumePath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, vol), snap.archiveSubPath()) + klog.V(2).Infof("Removing snapshot archive at %v", internalVolumePath) + if err = os.RemoveAll(internalVolumePath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to delete subdirectory: %v", err.Error()) + } + + return &csi.DeleteSnapshotResponse{}, nil } func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { @@ -325,6 +473,47 @@ func (cs *ControllerServer) internalUnmount(ctx context.Context, vol *nfsVolume) return err } +func (cs *ControllerServer) copyFromSnapshot(ctx context.Context, req *csi.CreateVolumeRequest, dstVol *nfsVolume) error { + snap, err := getNfsSnapFromID(req.VolumeContentSource.GetSnapshot().GetSnapshotId()) + if err != nil { + return status.Error(codes.NotFound, err.Error()) + } + snapVol := volumeFromSnapshot(snap) + + var volCap *csi.VolumeCapability + if len(req.GetVolumeCapabilities()) > 0 { + volCap = req.GetVolumeCapabilities()[0] + } + + if err = cs.internalMount(ctx, snapVol, nil, volCap); err != nil { + return status.Errorf(codes.Internal, "failed to mount src nfs server for snapshot volume copy: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, snapVol); err != nil { + klog.Warningf("failed to unmount src nfs server after snapshot volume copy: %v", err) + } + }() + if err = cs.internalMount(ctx, dstVol, nil, volCap); err != nil { + return status.Errorf(codes.Internal, "failed to mount dst nfs server for snapshot volume copy: %v", err) + } + defer func() { + if err = cs.internalUnmount(ctx, dstVol); err != nil { + klog.Warningf("failed to unmount dst nfs server after snapshot volume copy: %v", err) + } + }() + + // untar snapshot archive to dst path + snapPath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, snapVol), snap.archivePath()) + dstPath := getInternalVolumePath(cs.Driver.workingMountDir, dstVol) + klog.V(2).Infof("copy volume from snapshot %v -> %v", snapPath, dstPath) + out, err := exec.Command("tar", "-xzvf", snapPath, "-C", dstPath).CombinedOutput() + if err != nil { + return status.Errorf(codes.Internal, "failed to copy volume for snapshot: %v: %v", err, string(out)) + } + klog.V(2).Infof("volume copied from snapshot %v -> %v", snapPath, dstPath) + return nil +} + func (cs *ControllerServer) copyFromVolume(ctx context.Context, req *csi.CreateVolumeRequest, dstVol *nfsVolume) error { srcVol, err := getNfsVolFromID(req.GetVolumeContentSource().GetVolume().GetVolumeId()) if err != nil { @@ -340,26 +529,26 @@ func (cs *ControllerServer) copyFromVolume(ctx context.Context, req *csi.CreateV volCap = req.GetVolumeCapabilities()[0] } if err = cs.internalMount(ctx, srcVol, nil, volCap); err != nil { - return status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err.Error()) + return status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err) } defer func() { if err = cs.internalUnmount(ctx, srcVol); err != nil { - klog.Warningf("failed to unmount nfs server: %v", err.Error()) + klog.Warningf("failed to unmount nfs server: %v", err) } }() if err = cs.internalMount(ctx, dstVol, nil, volCap); err != nil { - return status.Errorf(codes.Internal, "failed to mount dst nfs server: %v", err.Error()) + return status.Errorf(codes.Internal, "failed to mount dst nfs server: %v", err) } defer func() { if err = cs.internalUnmount(ctx, dstVol); err != nil { - klog.Warningf("failed to unmount dst nfs server: %v", err.Error()) + klog.Warningf("failed to unmount dst nfs server: %v", err) } }() // recursive 'cp' with '-a' to handle symlinks out, err := exec.Command("cp", "-a", srcPath, dstPath).CombinedOutput() if err != nil { - return status.Error(codes.Internal, fmt.Sprintf("%v: %v", err, string(out))) + return status.Errorf(codes.Internal, "failed to copy volume %v: %v", err, string(out)) } klog.V(2).Infof("copied %s -> %s", srcPath, dstPath) return nil @@ -369,7 +558,7 @@ func (cs *ControllerServer) copyVolume(ctx context.Context, req *csi.CreateVolum vs := req.VolumeContentSource switch vs.Type.(type) { case *csi.VolumeContentSource_Snapshot: - return status.Error(codes.Unimplemented, "Currently only volume copy from another volume is supported") + return cs.copyFromSnapshot(ctx, req, vol) case *csi.VolumeContentSource_Volume: return cs.copyFromVolume(ctx, req, vol) default: @@ -377,6 +566,40 @@ func (cs *ControllerServer) copyVolume(ctx context.Context, req *csi.CreateVolum } } +// newNFSSnapshot Convert VolumeSnapshot parameters to a nfsSnapshot +func newNFSSnapshot(name string, params map[string]string, vol *nfsVolume) (*nfsSnapshot, error) { + server := vol.server + baseDir := vol.baseDir + for k, v := range params { + switch strings.ToLower(k) { + case paramServer: + server = v + case paramShare: + baseDir = v + } + } + + if server == "" { + return nil, fmt.Errorf("%v is a required parameter", paramServer) + } + snapshot := &nfsSnapshot{ + server: server, + baseDir: baseDir, + uuid: name, + } + if vol.subDir != "" { + snapshot.src = vol.subDir + } + if vol.uuid != "" { + snapshot.src = vol.uuid + } + if snapshot.src == "" { + return nil, fmt.Errorf("missing required source volume name") + } + snapshot.id = getSnapshotIDFromNfsSnapshot(snapshot) + return snapshot, nil +} + // newNFSVolume Convert VolumeCreate parameters to an nfsVolume func newNFSVolume(name string, size int64, params map[string]string, defaultOnDeletePolicy string) (*nfsVolume, error) { var server, baseDir, subDir, onDelete string @@ -470,6 +693,17 @@ func getVolumeIDFromNfsVol(vol *nfsVolume) string { return strings.Join(idElements, separator) } +// Given a nfsSnapshot, return a CSI snapshot id. +func getSnapshotIDFromNfsSnapshot(snap *nfsSnapshot) string { + idElements := make([]string, totalIDSnapElements) + idElements[idSnapServer] = strings.Trim(snap.server, "/") + idElements[idSnapBaseDir] = strings.Trim(snap.baseDir, "/") + idElements[idSnapUUID] = snap.uuid + idElements[idSnapArchivePath] = snap.uuid + idElements[idSnapArchiveName] = snap.src + return strings.Join(idElements, separator) +} + // Given a CSI volume id, return a nfsVolume // sample volume Id: // @@ -513,6 +747,25 @@ func getNfsVolFromID(id string) (*nfsVolume, error) { }, nil } +// Given a CSI snapshot ID, return a nfsSnapshot +// sample snapshot ID: +// +// nfs-server.default.svc.cluster.local#share#snapshot-016f784f-56f4-44d1-9041-5f59e82dbce1#snapshot-016f784f-56f4-44d1-9041-5f59e82dbce1#pvc-4bcbf944-b6f7-4bd0-b50f-3c3dd00efc64 +func getNfsSnapFromID(id string) (*nfsSnapshot, error) { + segments := strings.Split(id, separator) + if len(segments) == totalIDSnapElements { + return &nfsSnapshot{ + id: id, + server: segments[idSnapServer], + baseDir: segments[idSnapBaseDir], + src: segments[idSnapArchiveName], + uuid: segments[idSnapUUID], + }, nil + } + + return &nfsSnapshot{}, fmt.Errorf("failed to create nfsSnapshot from snapshot ID") +} + // isValidVolumeCapabilities validates the given VolumeCapability array is valid func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error { if len(volCaps) == 0 { @@ -525,3 +778,32 @@ func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error { } return nil } + +// Validate snapshot after internal mount +func validateSnapshot(snapInternalVolPath string, snap *nfsSnapshot) error { + return filepath.WalkDir(snapInternalVolPath, func(path string, d fs.DirEntry, err error) error { + if path == snapInternalVolPath { + // skip root + return nil + } + if err != nil { + return err + } + if d.Name() != snap.archiveName() { + // there should be just one archive in the snapshot path and archive name should match + return status.Errorf(codes.AlreadyExists, "snapshot with the same name but different source volume ID already exists: found %q, desired %q", d.Name(), snap.archiveName()) + } + return nil + }) +} + +// Volume for snapshot internal mount/unmount +func volumeFromSnapshot(snap *nfsSnapshot) *nfsVolume { + return &nfsVolume{ + id: snap.id, + server: snap.server, + baseDir: snap.baseDir, + subDir: snap.baseDir, + uuid: snap.uuid, + } +} diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index 8230bbf28..e312f05c1 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -88,6 +88,7 @@ func NewDriver(options *DriverOptions) *Driver { csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, + csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, }) n.AddNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{ From 8694d473213f639cf8616719c61562bea90a37a3 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Sun, 26 Mar 2023 20:26:21 +0200 Subject: [PATCH 4/5] test: copy volume from snapshot unit tests --- pkg/nfs/controllerserver_test.go | 314 ++++++++++++++++++++++++++++++- 1 file changed, 311 insertions(+), 3 deletions(-) diff --git a/pkg/nfs/controllerserver_test.go b/pkg/nfs/controllerserver_test.go index 08663510e..8e5d43331 100644 --- a/pkg/nfs/controllerserver_test.go +++ b/pkg/nfs/controllerserver_test.go @@ -17,6 +17,8 @@ limitations under the License. package nfs import ( + "archive/tar" + "compress/gzip" "os" "path/filepath" "reflect" @@ -31,6 +33,7 @@ import ( "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" mount "k8s.io/mount-utils" ) @@ -352,6 +355,13 @@ func TestControllerGetCapabilities(t *testing.T) { }, }, }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + }, + }, + }, }, }, expectedErr: nil, @@ -687,6 +697,8 @@ func TestCopyVolume(t *testing.T) { req *csi.CreateVolumeRequest dstVol *nfsVolume expectErr bool + prepare func() error + cleanup func() error }{ { desc: "copy volume from valid volume", @@ -707,6 +719,56 @@ func TestCopyVolume(t *testing.T) { subDir: "subdir", uuid: "dst-pv-name", }, + prepare: func() error { return os.MkdirAll("/tmp/src-pv-name/subdir", 0777) }, + cleanup: func() error { return os.RemoveAll("/tmp/src-pv-name") }, + }, + { + desc: "copy volume from valid snapshot", + req: &csi.CreateVolumeRequest{ + Name: "snapshot-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + }, + }, + }, + }, + dstVol: &nfsVolume{ + id: "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name", + server: "//nfs-server.default.svc.cluster.local", + baseDir: "share", + subDir: "subdir", + uuid: "dst-pv-name", + }, + prepare: func() error { + if err := os.MkdirAll("/tmp/snapshot-name/share/snapshot-name/", 0777); err != nil { + return err + } + file, err := os.Create("/tmp/snapshot-name/share/snapshot-name/src-pv-name.tar.gz") + if err != nil { + return err + } + defer file.Close() + gzipWriter := gzip.NewWriter(file) + defer gzipWriter.Close() + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + body := "test file" + hdr := &tar.Header{ + Name: "test.txt", + Mode: 0777, + Size: int64(len(body)), + } + if err := tarWriter.WriteHeader(hdr); err != nil { + return err + } + if _, err := tarWriter.Write([]byte(body)); err != nil { + return err + } + return nil + }, + cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") }, }, { desc: "copy volume missing source id", @@ -747,17 +809,263 @@ func TestCopyVolume(t *testing.T) { }, expectErr: true, }, - } - if err := os.MkdirAll("/tmp/src-pv-name/subdir", 0777); err != nil { - t.Fatalf("Unexpected error when creating srcVolume: %v", err) + { + desc: "copy volume from broken snapshot", + req: &csi.CreateVolumeRequest{ + Name: "snapshot-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + }, + }, + }, + }, + dstVol: &nfsVolume{ + id: "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name", + server: "//nfs-server.default.svc.cluster.local", + baseDir: "share", + subDir: "subdir", + uuid: "dst-pv-name", + }, + expectErr: true, + }, + { + desc: "copy volume from missing snapshot", + req: &csi.CreateVolumeRequest{ + Name: "snapshot-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{}, + }, + }, + }, + dstVol: &nfsVolume{ + id: "nfs-server.default.svc.cluster.local#share#subdir#dst-pv-name", + server: "//nfs-server.default.svc.cluster.local", + baseDir: "share", + subDir: "subdir", + uuid: "dst-pv-name", + }, + expectErr: true, + }, + { + desc: "copy volume from snapshot into missing dst volume", + req: &csi.CreateVolumeRequest{ + Name: "snapshot-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{}, + }, + }, + }, + dstVol: &nfsVolume{ + server: "//nfs-server.default.svc.cluster.local", + baseDir: "share", + subDir: "subdir", + uuid: "dst-pv-name", + }, + expectErr: true, + }, } for _, test := range cases { t.Run(test.desc, func(t *testing.T) { + if test.prepare != nil { + if err := test.prepare(); err != nil { + t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err) + } + } cs := initTestController(t) err := cs.copyVolume(context.TODO(), test.req, test.dstVol) if (err == nil) == test.expectErr { t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err) } + if test.cleanup != nil { + if err := test.cleanup(); err != nil { + t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err) + } + } }) } } + +func TestCreateSnapshot(t *testing.T) { + cases := []struct { + desc string + req *csi.CreateSnapshotRequest + expResp *csi.CreateSnapshotResponse + expectErr bool + prepare func() error + cleanup func() error + }{ + { + desc: "create snapshot with valid request", + req: &csi.CreateSnapshotRequest{ + SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name", + Name: "snapshot-name", + }, + expResp: &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name", + ReadyToUse: true, + SizeBytes: 1, // doesn't match exact size, just denotes non-zero size expected + CreationTime: timestamppb.Now(), // doesn't match exact timestamp, just denotes non-zero ts expected + }, + }, + prepare: func() error { return os.MkdirAll("/tmp/src-pv-name/subdir", 0777) }, + cleanup: func() error { return os.RemoveAll("/tmp/src-pv-name") }, + }, + { + desc: "create snapshot from nonexisting volume", + req: &csi.CreateSnapshotRequest{ + SourceVolumeId: "nfs-server.default.svc.cluster.local#share#subdir#src-pv-name", + Name: "snapshot-name", + }, + expectErr: true, + }, + } + for _, test := range cases { + t.Run(test.desc, func(t *testing.T) { + if test.prepare != nil { + if err := test.prepare(); err != nil { + t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err) + } + } + cs := initTestController(t) + resp, err := cs.CreateSnapshot(context.TODO(), test.req) + if (err == nil) == test.expectErr { + t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err) + } + if err := matchCreateSnapshotResponse(test.expResp, resp); err != nil { + t.Errorf("[test: %s] failed %q: got resp %+v, expected %+v", test.desc, err, resp, test.expResp) + } + if test.cleanup != nil { + if err := test.cleanup(); err != nil { + t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err) + } + } + }) + } +} + +func TestDeleteSnapshot(t *testing.T) { + cases := []struct { + desc string + req *csi.DeleteSnapshotRequest + expResp *csi.DeleteSnapshotResponse + expectErr bool + prepare func() error + cleanup func() error + }{ + { + desc: "delete valid snapshot", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + }, + expResp: &csi.DeleteSnapshotResponse{}, + prepare: func() error { + if err := os.MkdirAll("/tmp/snapshot-name/snapshot-name/", 0777); err != nil { + return err + } + f, err := os.OpenFile("/tmp/snapshot-name/snapshot-name/src-pv-name.tar.gz", os.O_CREATE, 0777) + if err != nil { + return err + } + return f.Close() + }, + cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") }, + }, + { + desc: "delete nonexisting snapshot", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + }, + expResp: &csi.DeleteSnapshotResponse{}, + }, + { + desc: "delete snapshot with improper id", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: "incorrect-snap-id", + }, + expResp: &csi.DeleteSnapshotResponse{}, + }, + { + desc: "delete valid snapshot with mount options", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: "nfs-server.default.svc.cluster.local#share#snapshot-name#snapshot-name#src-pv-name", + Secrets: map[string]string{"mountoptions": "nfsvers=4.1"}, + }, + expResp: &csi.DeleteSnapshotResponse{}, + prepare: func() error { + if err := os.MkdirAll("/tmp/snapshot-name/snapshot-name/", 0777); err != nil { + return err + } + f, err := os.OpenFile("/tmp/snapshot-name/snapshot-name/src-pv-name.tar.gz", os.O_CREATE, 0777) + if err != nil { + return err + } + return f.Close() + }, + cleanup: func() error { return os.RemoveAll("/tmp/snapshot-name") }, + }, + } + for _, test := range cases { + t.Run(test.desc, func(t *testing.T) { + if test.prepare != nil { + if err := test.prepare(); err != nil { + t.Errorf(`[test: %s] prepare failed: "%v"`, test.desc, err) + } + } + cs := initTestController(t) + resp, err := cs.DeleteSnapshot(context.TODO(), test.req) + if (err == nil) == test.expectErr { + t.Errorf(`[test: %s] Error expectation mismatch, expected error: "%v", received: %q`, test.desc, test.expectErr, err) + } + if !reflect.DeepEqual(test.expResp, resp) { + t.Errorf("[test: %s] got resp %+v, expected %+v", test.desc, resp, test.expResp) + } + if test.cleanup != nil { + if err := test.cleanup(); err != nil { + t.Errorf(`[test: %s] cleanup failed: "%v"`, test.desc, err) + } + } + }) + } +} + +func matchCreateSnapshotResponse(e, r *csi.CreateSnapshotResponse) error { + if e == nil && r == nil { + return nil + } + if e == nil || e.Snapshot == nil { + return fmt.Errorf("expected nil response") + } + if r == nil || r.Snapshot == nil { + return fmt.Errorf("unexpected nil response") + } + es, rs := e.Snapshot, r.Snapshot + + var errs []string + // comparing ts and size just for presence, not the exact value + if es.CreationTime.IsValid() != rs.CreationTime.IsValid() { + errs = append(errs, "CreationTime") + } + if (es.SizeBytes == 0) != (rs.SizeBytes == 0) { + errs = append(errs, "SizeBytes") + } + // comparing remaining fields for exact match + if es.ReadyToUse != rs.ReadyToUse { + errs = append(errs, "ReadyToUse") + } + if es.SnapshotId != rs.SnapshotId { + errs = append(errs, "SnapshotId") + } + if es.SourceVolumeId != rs.SourceVolumeId { + errs = append(errs, "SourceVolumeId") + } + if len(errs) == 0 { + return nil + } + return fmt.Errorf("mismatch CreateSnapshotResponse in fields: %v", strings.Join(errs, ", ")) +} From 6881ee5ddffbc00b09d512ab0ba34ed3c64b0658 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Sat, 18 Mar 2023 19:33:43 +0100 Subject: [PATCH 5/5] chore: go mod $ go mod tidy && go mod vendor --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5140a414f..ea4a98d05 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/stretchr/testify v1.8.0 golang.org/x/net v0.7.0 google.golang.org/grpc v1.40.0 + google.golang.org/protobuf v1.27.1 k8s.io/api v0.23.14 k8s.io/apimachinery v0.23.14 k8s.io/client-go v0.23.14 @@ -82,7 +83,6 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect - google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect