From d113150bddff3bb2817073656b94a0d2a24ba675 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Thu, 29 Sep 2022 13:39:17 +0000 Subject: [PATCH] doc: cut v1.22.0 release --- README.md | 4 +- charts/README.md | 2 +- charts/index.yaml | 73 +- charts/latest/azurefile-csi-driver-v0.0.0.tgz | Bin 11415 -> 0 bytes .../latest/azurefile-csi-driver-v1.22.0.tgz | Bin 0 -> 11399 bytes charts/latest/azurefile-csi-driver/Chart.yaml | 4 +- .../latest/azurefile-csi-driver/values.yaml | 4 +- .../v1.22.0/azurefile-csi-driver-v1.22.0.tgz | Bin 0 -> 11399 bytes .../v1.22.0/azurefile-csi-driver/Chart.yaml | 5 + .../azurefile-csi-driver/templates/NOTES.txt | 5 + .../templates/_helpers.tpl | 49 ++ .../templates/crd-csi-snapshot.yaml | 661 ++++++++++++++++++ .../templates/csi-azurefile-controller.yaml | 239 +++++++ .../templates/csi-azurefile-driver.yaml | 17 + .../templates/csi-azurefile-node-windows.yaml | 222 ++++++ .../templates/csi-azurefile-node.yaml | 217 ++++++ .../templates/csi-snapshot-controller.yaml | 65 ++ .../rbac-csi-azurefile-controller.yaml | 207 ++++++ .../templates/rbac-csi-azurefile-node.yaml | 29 + .../rbac-csi-snapshot-controller.yaml | 80 +++ ...rviceaccount-csi-azurefile-controller.yaml | 9 + .../serviceaccount-csi-azurefile-node.yaml | 9 + ...erviceaccount-csi-snapshot-controller.yaml | 9 + .../v1.22.0/azurefile-csi-driver/values.yaml | 254 +++++++ deploy/csi-azurefile-controller.yaml | 2 +- deploy/csi-azurefile-node-windows.yaml | 2 +- deploy/csi-azurefile-node.yaml | 2 +- deploy/v1.22.0/crd-csi-snapshot.yaml | 659 +++++++++++++++++ deploy/v1.22.0/csi-azurefile-controller.yaml | 180 +++++ deploy/v1.22.0/csi-azurefile-driver.yaml | 15 + .../v1.22.0/csi-azurefile-node-windows.yaml | 181 +++++ deploy/v1.22.0/csi-azurefile-node.yaml | 157 +++++ deploy/v1.22.0/csi-snapshot-controller.yaml | 46 ++ .../rbac-csi-azurefile-controller.yaml | 194 +++++ deploy/v1.22.0/rbac-csi-azurefile-node.yaml | 30 + .../v1.22.0/rbac-csi-snapshot-controller.yaml | 78 +++ docs/install-azurefile-csi-driver.md | 2 +- docs/install-csi-driver-v1.22.0.md | 45 ++ 38 files changed, 3714 insertions(+), 43 deletions(-) delete mode 100644 charts/latest/azurefile-csi-driver-v0.0.0.tgz create mode 100644 charts/latest/azurefile-csi-driver-v1.22.0.tgz create mode 100644 charts/v1.22.0/azurefile-csi-driver-v1.22.0.tgz create mode 100644 charts/v1.22.0/azurefile-csi-driver/Chart.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/NOTES.txt create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/_helpers.tpl create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/crd-csi-snapshot.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-controller.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-driver.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-node-windows.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-node.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/csi-snapshot-controller.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-azurefile-controller.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-snapshot-controller.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-azurefile-controller.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-azurefile-node.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-snapshot-controller.yaml create mode 100644 charts/v1.22.0/azurefile-csi-driver/values.yaml create mode 100644 deploy/v1.22.0/crd-csi-snapshot.yaml create mode 100644 deploy/v1.22.0/csi-azurefile-controller.yaml create mode 100644 deploy/v1.22.0/csi-azurefile-driver.yaml create mode 100644 deploy/v1.22.0/csi-azurefile-node-windows.yaml create mode 100644 deploy/v1.22.0/csi-azurefile-node.yaml create mode 100644 deploy/v1.22.0/csi-snapshot-controller.yaml create mode 100644 deploy/v1.22.0/rbac-csi-azurefile-controller.yaml create mode 100644 deploy/v1.22.0/rbac-csi-azurefile-node.yaml create mode 100644 deploy/v1.22.0/rbac-csi-snapshot-controller.yaml create mode 100644 docs/install-csi-driver-v1.22.0.md diff --git a/README.md b/README.md index 458dd5d44e..4ef4b57b36 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@ This driver allows Kubernetes to use [Azure File](https://docs.microsoft.com/en- ### Container Images & Kubernetes Compatibility: |Driver Version |Image | supported k8s version | |----------------|---------------------------------------------------------- |-----------------------| -|master branch |mcr.microsoft.com/k8s/csi/azurefile-csi:latest | 1.20+ | +|master branch |mcr.microsoft.com/k8s/csi/azurefile-csi:latest | 1.21+ | +|v1.22.0 |mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.22.0 | 1.21+ | |v1.21.0 |mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.21.0 | 1.20+ | |v1.20.0 |mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.20.0 | 1.20+ | -|v1.19.0 |mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.19.0 | 1.20+ | ### Driver parameters Please refer to [driver parameters](./docs/driver-parameters.md) diff --git a/charts/README.md b/charts/README.md index dc95d5e006..89c96f08fd 100644 --- a/charts/README.md +++ b/charts/README.md @@ -16,7 +16,7 @@ ### install a specific version ```console helm repo add azurefile-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/charts -helm install azurefile-csi-driver azurefile-csi-driver/azurefile-csi-driver --namespace kube-system --version v1.21.0 +helm install azurefile-csi-driver azurefile-csi-driver/azurefile-csi-driver --namespace kube-system --version v1.22.0 ``` ### install on RedHat/CentOS diff --git a/charts/index.yaml b/charts/index.yaml index 159b86ff58..6716181735 100644 --- a/charts/index.yaml +++ b/charts/index.yaml @@ -1,9 +1,27 @@ apiVersion: v1 entries: azurefile-csi-driver: + - apiVersion: v1 + appVersion: v1.22.0 + created: "2022-09-29T13:38:36.272180768Z" + description: Azure File Container Storage Interface (CSI) Storage Plugin + digest: 389c94d9060565e8aa67182dbb07718b11b3fe371903a5c7c135dc3557b8ce4c + name: azurefile-csi-driver + urls: + - https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/charts/latest/azurefile-csi-driver-v1.22.0.tgz + version: v1.22.0 + - apiVersion: v1 + appVersion: v1.22.0 + created: "2022-09-29T13:38:36.302075708Z" + description: Azure File Container Storage Interface (CSI) Storage Plugin + digest: 253d87a8b876dbdd55870a7fee88547393179d03f193661125f5a0b63411f922 + name: azurefile-csi-driver + urls: + - https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/charts/v1.22.0/azurefile-csi-driver-v1.22.0.tgz + version: v1.22.0 - apiVersion: v1 appVersion: v1.21.0 - created: "2022-09-15T12:43:56.863125912Z" + created: "2022-09-29T13:38:36.300209593Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: d45bf3455ebadc9cc5afaf9da66aa1ea1d4b719cfdff5af661f93bb26c01a504 name: azurefile-csi-driver @@ -12,7 +30,7 @@ entries: version: v1.21.0 - apiVersion: v1 appVersion: v1.20.0 - created: "2022-09-15T12:43:56.861928496Z" + created: "2022-09-29T13:38:36.298731081Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 7cc43d57a79137aea5414fb51a9bbd77bb679b29ee49c06865c1a5b9ba60be99 name: azurefile-csi-driver @@ -21,7 +39,7 @@ entries: version: v1.20.0 - apiVersion: v1 appVersion: v1.19.0 - created: "2022-09-15T12:43:56.859934768Z" + created: "2022-09-29T13:38:36.296286661Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 18f6efbed424efd661fde43be2e5a48a5012a46a7938c33b36963cbd9875a5af name: azurefile-csi-driver @@ -30,7 +48,7 @@ entries: version: v1.19.0 - apiVersion: v1 appVersion: v1.18.0 - created: "2022-09-15T12:43:56.856832026Z" + created: "2022-09-29T13:38:36.293766441Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 696ca23d9ee517f71ef5e852955c8d0f1017c331c025426c6fcbe7a06d006c66 name: azurefile-csi-driver @@ -39,7 +57,7 @@ entries: version: v1.18.0 - apiVersion: v1 appVersion: v1.17.0 - created: "2022-09-15T12:43:56.855703911Z" + created: "2022-09-29T13:38:36.292189528Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 5632f61265a3b78dce3e2b15e07cc9b14a7f54a778878c02ca2d9fe69ca0344e name: azurefile-csi-driver @@ -48,7 +66,7 @@ entries: version: v1.17.0 - apiVersion: v1 appVersion: v1.16.0 - created: "2022-09-15T12:43:56.854564795Z" + created: "2022-09-29T13:38:36.290497315Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: a7a2d57e8eca7dc06c8b2cffb9bccb857753eb110b8e70760b3e04f4e6a87552 name: azurefile-csi-driver @@ -57,7 +75,7 @@ entries: version: v1.16.0 - apiVersion: v1 appVersion: v1.15.0 - created: "2022-09-15T12:43:56.85276427Z" + created: "2022-09-29T13:38:36.288725101Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: c1a31dadce233a90c19dce70f6cc92ba2e20bbaa1b1883baea72381d09303118 name: azurefile-csi-driver @@ -66,7 +84,7 @@ entries: version: v1.15.0 - apiVersion: v1 appVersion: v1.14.0 - created: "2022-09-15T12:43:56.850802044Z" + created: "2022-09-29T13:38:36.287116588Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 0c9ad4afa5ebfdb2851ad93eb16a0382d61448714b7556899360730a2fdf463a name: azurefile-csi-driver @@ -75,7 +93,7 @@ entries: version: v1.14.0 - apiVersion: v1 appVersion: v1.13.0 - created: "2022-09-15T12:43:56.848542213Z" + created: "2022-09-29T13:38:36.285484375Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 214042b029d858b50a0f8bba33a7aa2b41d1b67bce16f957ca183ae7438dac3f name: azurefile-csi-driver @@ -84,7 +102,7 @@ entries: version: v1.13.0 - apiVersion: v1 appVersion: v1.12.0 - created: "2022-09-15T12:43:56.843405843Z" + created: "2022-09-29T13:38:36.280489635Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: fbd63929671066a26df898d32282a6e79c39499a39c71761c546d069459d847d name: azurefile-csi-driver @@ -93,7 +111,7 @@ entries: version: v1.12.0 - apiVersion: v1 appVersion: v1.11.0 - created: "2022-09-15T12:43:56.840375001Z" + created: "2022-09-29T13:38:36.278904922Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 76bd438f8391d08235b09fbca859f25a9fcf8e018fd1e7e33444ca9ea946ce4b name: azurefile-csi-driver @@ -102,7 +120,7 @@ entries: version: v1.11.0 - apiVersion: v1 appVersion: v1.10.0 - created: "2022-09-15T12:43:56.838641178Z" + created: "2022-09-29T13:38:36.276571003Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 845a9de8b571b255d05ae9c643d9b90a57fe6507ff3fb735c88b41f99f6f28dc name: azurefile-csi-driver @@ -111,7 +129,7 @@ entries: version: v1.10.0 - apiVersion: v1 appVersion: v1.9.0 - created: "2022-09-15T12:43:56.871722229Z" + created: "2022-09-29T13:38:36.312517491Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 59a8057fbbd6d59919b84ef0c3396d5a0a46d1f29df604457db676f4af63c714 name: azurefile-csi-driver @@ -120,7 +138,7 @@ entries: version: v1.9.0 - apiVersion: v1 appVersion: v1.8.0 - created: "2022-09-15T12:43:56.870417512Z" + created: "2022-09-29T13:38:36.310824278Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 455b7c342194311046df526d10926d94f3bef24f753a07003fb1cad211c4cb52 name: azurefile-csi-driver @@ -129,7 +147,7 @@ entries: version: v1.8.0 - apiVersion: v1 appVersion: v1.7.0 - created: "2022-09-15T12:43:56.869034393Z" + created: "2022-09-29T13:38:36.309212265Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 057b3f6ef6001d3457fbffc27f90316c981a089696abd3d38bcc8de5537dfa6f name: azurefile-csi-driver @@ -138,7 +156,7 @@ entries: version: v1.7.0 - apiVersion: v1 appVersion: v1.6.0 - created: "2022-09-15T12:43:56.867722875Z" + created: "2022-09-29T13:38:36.307880354Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: cc2a0dda824cdda4e8141e26878bbb481c5a52e45785a5dbf72e54f2a376e522 name: azurefile-csi-driver @@ -147,7 +165,7 @@ entries: version: v1.6.0 - apiVersion: v1 appVersion: v1.5.0 - created: "2022-09-15T12:43:56.866480858Z" + created: "2022-09-29T13:38:36.306551044Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 2258177477415ddecd83dc46dfd88833223623224c7fe396590b617082bcd845 name: azurefile-csi-driver @@ -156,7 +174,7 @@ entries: version: v1.5.0 - apiVersion: v1 appVersion: v1.4.0 - created: "2022-09-15T12:43:56.864693333Z" + created: "2022-09-29T13:38:36.304395226Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 40e9bc4ee187789166fcb7c3c82b85b33ecd3a6096266fe74e411d6b48961ece name: azurefile-csi-driver @@ -165,7 +183,7 @@ entries: version: v1.4.0 - apiVersion: v1 appVersion: v1.3.0 - created: "2022-09-15T12:43:56.863798321Z" + created: "2022-09-29T13:38:36.303126216Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 12942f422b7cccbfe950bbdbd5c844f5ae4b7c292f32389cba312730a6fe9a62 name: azurefile-csi-driver @@ -174,7 +192,7 @@ entries: version: v1.3.0 - apiVersion: v1 appVersion: v1.2.0 - created: "2022-09-15T12:43:56.860589177Z" + created: "2022-09-29T13:38:36.297168568Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: b62f44b757416a9e1f5a91e19285f5f5056ec6068802dd9cd82373bef40c9ee9 name: azurefile-csi-driver @@ -183,7 +201,7 @@ entries: version: v1.2.0 - apiVersion: v1 appVersion: v1.1.0 - created: "2022-09-15T12:43:56.836826853Z" + created: "2022-09-29T13:38:36.274676888Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 675d96b309a1c5c491053ebbb854c046737420929c4f0692839afdaaf0db3933 name: azurefile-csi-driver @@ -192,20 +210,11 @@ entries: version: v1.1.0 - apiVersion: v1 appVersion: v1.0.0 - created: "2022-09-15T12:43:56.836078843Z" + created: "2022-09-29T13:38:36.27363418Z" description: Azure File Container Storage Interface (CSI) Storage Plugin digest: 6fd5e54e949ef1061a08d5477bc580204c91dde8f01da195e95dd60ade209492 name: azurefile-csi-driver urls: - https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/charts/v1.0.0/azurefile-csi-driver-v1.0.0.tgz version: v1.0.0 - - apiVersion: v1 - appVersion: latest - created: "2022-09-15T12:43:56.835293932Z" - description: Azure File Container Storage Interface (CSI) Storage Plugin - digest: 2f1513871466db9af0eb496c506de40fc6daf244163dfab040479834ed1ef644 - name: azurefile-csi-driver - urls: - - https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/charts/latest/azurefile-csi-driver-v0.0.0.tgz - version: v0.0.0 -generated: "2022-09-15T12:43:56.833010501Z" +generated: "2022-09-29T13:38:36.270257953Z" diff --git a/charts/latest/azurefile-csi-driver-v0.0.0.tgz b/charts/latest/azurefile-csi-driver-v0.0.0.tgz deleted file mode 100644 index 4b41c890d6800d9bd1a9e60910c34575a12c7422..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11415 zcmZ8{Q*b4~7Hu@a#L2|w#I|kQwllGDVo!{TZQJ(A#I|kYyt%L5ty}fFy1J^@>ixI7 zdiUCkC>r($*nb9y8jRLhN`=W(N}fZ`i<{kqRfEY)mEBrLm77CRLxV$J)6T}&!OTld z*^yt$+|C~C!td2(o$JXGRMiVnJf)V1qna8&#JVOU?}hI%lGSwCkbRYX`EUfEq=+^` z-UlwUdgl7|Opo7%tQ;QinzUVLiaH8;!@%cz6A=*06%&aZM;o#4%mUft$2P2obi{Sj zyFSE>923Txh@U^QGJg4*Elxn10XG z{_=P-Ai)1|2wDUx`~)H2CV(j#Er_1r!nXw`#47FF2@B}~q3|YyCW8rOiZw9ABSgob ze#gv6TD>1Rnyh1(@s7%Lqz@&D#g_FBjy=z0C~myMX2&RI@EsEeR;1MsAOD$-P=V~D z^)?o<+ZJ2yqn1LcfL|;}FG-nWu=D|J_ScE0g{zo&bjSs)ERDH6Rj6BH5s%eQJ`Br; z%4zc2cifF5P{BF_SKSu9)gLf5;nY*)0y+VJ;ZNnGe%)#=1|yM>1J6k2iV2NNF20(a z!ktB<(v0A(M#~e3k-nggKyuL-2owBoF_36GzY;_~eG#e9cJkM$l{Rtn(TiRU%P6^G zB&}%7sU+5MWMD$J9Uy}sc3@*`b%MJSCcfuZ?m z?d4QP`~IbnIGJ1~6q)%uy&VD=tQ*;Tg1W(R-u>&QEmo|9tg&?SyKJuw*aW#C#%+V)W!87j=1MDY;L zvc5rLs#KR*tiDKgE*uM#B=oR!fFQo`P>~GYI%g<}jG(_@yHx_rhf!1~$TfOgK4izb z{^W&C@zP=J5o- zw&`@PPR#Hf%W>hjC*kdB8lX?R^0k}>(xyN6zcI9VRljfQR~i-@>Qzd-`BAX%Gv|u| ztJ$a){on-J5}n_Z%mkchSvs^~Hv4=--gKs2js}Sx2O#N>{A3~E)$JE`KreBy)O%P= zgaBH}vVaMOO={2I$e$XQy4e^svCh}sT(++B^CP#j7XW`B5kHY#$s zZL{q)@xNMn5Wit;jnSy7aq)JjoH;DGCAWTj-p0whl^dMUh?Cf$a z*@5}=ocWB?@9_hX7$5i(@r!zfMb^TKP%0Jr`wM66dEEYj@8YIe2()q<@&- z9X=gD?WA}0b#`{o*JEiO6Qa9ma+`_ zfbs^)oHd<#)VrNbBOU25ujbH^9p<={HD#oC?wdeZV=iY;CnqPtEqz)*FEzYjr*>~I z0~Qa4bL?j`PlXd44ZUe7r}C~N{)phCiB0khh+{8Ap^BNp^%qM%-RtWM(uP#{IDO=5 zPE8<8Mn4x--&{7~b|^kwIK{Qn8SI!BZ2NZO(pU00x{jwUPRC89jiW?cOx^WR@LVUw zf@!M>x3Y%=Ts7LW7;!F9tiUuA=pgA_@oDr}UF7xT*Mkj(!MdSj9HvM6@`AZ7x6Uly zc5O=G0yX_hpL^CdsL!K0tC~R7_B|jeupVSLQ($$(`;OP0WC;vHfTiuJssB;QjVoQt~^&3j!|E zoV-Qp1QIjQQ%tgZg;I)$8!7fV(Xq!@7y$qG5PBR5-S4wPiaC}$u5RF?WJZ2mh5q5u zmWUfzjdnGPRsZvoE&(*7dLz1HLOi$_AB;5yy7HI)LN>U*bh`tg%%m}{2MDvG#U$Jv z{UD>VeWp~J-%~IS`D*xlw5=_TCfE*5_m)x+?(~a@eeBhb#hg3r@pVpsF%tEYEgEUzm1n&G_F>e8wz+7=b!SDBu@Oi?$rwI{r zhM&Dp${?436x4cRqaVXGifk&lX)HO7)@b3z{g$lCQH+L4bcctSj*ZEdhH z9QbcX79EQ=W|XRAP|n_dFLSlw;_73D#GIkX9yK-UF%0SP*mbRmL8b4(c>*8#1Tk)d z4Eds*r_jRTswz7Ap+H4CwCb;jpPVKvoa2VQPycQgB=hFdhc++LWH*3LUSp-9bmWLV z@nW0n22X~(`&eVWES%}?^PtObic3ae^qY;=I}<Z$p zBC7}cEW0R8%vC-6!>0iJMeij4PrL_xIP@hekjE2}LGRY&I%qSy8}JGI5*$)RX)(*T zmy(hV+~?}1Q}_`Mbw#e!ZA!TDFR~d`#@!kX$;<;^F3}gShG%_46;33OPw1Rtp)6P0Ro6z_ym=7V?S)xR zfwPx)1c%a6glMO9C>T8JLRU)0#lwM>d*=O+BM?W2;skR?MhXMVj-vx+y|G zK{^MFji5vbl#OCxoUdH4Dtfs5s9!mJF32+(uO(q)$D%?qJ0Z-k5j)S+D;l~|7L|vZ z=@7LrN@l1A1X-iDRmW|iM&YuEmSIKsQdRMo8H6R%{3j$jU5s@kPzg;89PL1pUqe&~ zEu8IxJKIN(Ti-jw1%6|wGP9C#yjI+T{;MZquW~@;heqG1$Pn|6>v0$Sk z6x63rudcj-kcNdyTguX5hDC#hOoMS0B~)joM<$pbEGF0lvQR78q?Dz7FekY!*F*va z2GnjQR*;I9v^bnh(tP#zq9Fv9{6F;@BfL}87Hbz(NPnjb0zt zA=xbc@qM4|sZKr^LwRCn;YsWr+sYa(jHmrACKqAIWvo}|{^;D)OfC}a6 z2LzvtBG1Y0>v+6RT6?at|JwI1QK*5pHjo!-oOd*oSbgXt`;XQIc?Q)p?!^xc411Ch zboW#_jNP4uGxZP=pvX?*f#OdVp#IeqDQCg4&0lv(8+T_3X&px}K3+w*bILaXR6aRV z|D_eJX67HNv0!dE+2EQS1Lo*5{_;BW1=StQ%dky-w5`*B>g}xa5$e#4tC2}Ft@{m3 zg$1c#g8wqh-fp6knvO1-3!^BL9z*Mk*N~03tvvlJcUp+iVO%_9$lF-<>Z-dGIKM(syTlM&N*1JC5C76$A-BbJpvctQyY051?wWx} zaxi?|e~`t(yrODO))6*Ygx$59I-I1MErWMs$I2NJ%@Tulxy3Z5Nl=)#L4D*bzVL)f*Mxs*MWTbD_PDK( zKnz8}kS3|9UEjb3c}zajvF59QYE5JaTCFG>32WZ1bi0Fhlx1M>SQ`1mygLH(>yc2* z1AR-I94wzpg#c65V{na>*!EZ~i()%R>vxPP0dDmtn{(6TEREUcG9V(IO?I9Z)Ozbf!0bh^iMz?zX`FV!s9<_(N%;P{`H z!G3&8uXHv+RqZn8z&e$)u^CB!>Bxpt`itjVER-zEB80n1_-;8g)dAhIUg^YwdLMl1 z@Z(%;=sPyKX&vpKoWzGLc{RYg>31_evHfT9KeW~5df0?TTiz0agv=Fo>38I0#vTjz zX2;HmcN+5c=ZKB@dS&hCza>ENWJKL!-GZcYw?uh|WQAr`{i(3G_g6_H^7zc_(m~~n$ zR#S=%ZVO>eNNZ(?xtyh3#11GGc|h3u4kkL@(`m+nJAZD!`Ib%&Asn0L zAIO2_23Z<<;U?%FrO>l;rUPBRoPMEue)lgMQf2uOI&phz4DBDLJp|7Z*pE4o-{WM~ z#_08~qW;iFK}fZ=yvxoV>P0z64i^i?9Ob|~yJ2kSZeeiG*<4!>=J%*2yTrF}ZWJ~q zTwSrMxaiz+U5FQ`#Ei21t5{sc3%vvGh~}MkZ*@Xes-f}=yFu(;hmvFQzUtyFP;#j|I0s4K}LB5(hLlxq-G={Ht^ z{)(SX$fiHpsBn-EKp`+kDRU31_=hL1g*H^JQ8KdFXz9NOn)gJ^4Y3}P7UWtmtEN}QJA?~7` z6Y8_YB^D&T62DHku`e|Jv@6r~F+h9!qH!>foKiU48Iv#pW!WIqCla&pGI2dPG(RhN zOjQ=RbI`$>vajq!(XA{Bo*1#BH74ic)y&`BS7}Ul<5XUU2mriITJ$Xaks1;IN}(Cz zosxWb%y(O)`4QHVb+oAq4BFqkZIQb2dXk4Gwo(y7;7g<^@$|BmJ-h=aiY3{oXUj&} zt@{=e#O29npyK;)ORY*}8`lXs1S*xHHf~bYNac$p%{)aK;R|n=|4_aPfTY_^S!%;V ze<6>j8ch7zj`G{-j`DvAB*Ml25&s280g}VHXxb6+GKDBBZY0S&*_h~pkJHu`ePP3!C6+h-F)qyIM4fW_^ek@el);|YuN^8cvifrj`X<6h46@b6RZ$=M4s)`1 zSZqHZqik2E44zc+WJ@0>z-a9f9Yff>3x$bsw2suISFFW3KE@lge^P*P`)~AbXQ7dwR zIQfl*+F9}^VD<4=z7q0Qd;X6;AJiwn^May#Q*(8Cn#W7X@wLLrlTw|U$tid zQ{l;y7u7|_KIPGqw_QR^A6yu&ecdf!S^I-lP z#YJ=RcIw|omf?l_X6!WJcee0_*s4Kzeo6x>cBR*GAP=2ef7e>I2Vr4irck#YCzHpN z&NnW$^$7@G3TOsZb<4iEX_HJ!OnrG_Ar1RF6cFbeQ4J6bMPt27&70;;e2HkiuQ|R2 zi=T(__ILLXQkAcu(czGXK2Cj=?#d6l_WM0v#-9IgIXo5fAiRYFlEoke1VTnm5zMTS z!Kqrz=7&(Uf2omvE(Z@rgp{zyll8R6K!n;JGQf+NtGkR2Qm_6v-9LwJNbTgSDRDAU zAK&wHPqDV&_hN&fki@o8wEXpp zkR&_72T`C9y&l*N>e!!gvvL5OGYFMOn&-NUwg=t@a>ScO3a8_l#Pg!biw-&|z?=6} z&&%+Neq&1>UkiMT;UPrj{EWdKNe5Y2UzorvQnviH@5>ozKHsQ5SEE7@Xy7(!d?^uo z4C^8UZ2oYdQSRY7csSU|U_ePh6!V+$M37m`@LZ6H9l~YX3Q7ygS`4|HfY8PRuD2=LE5z*a$&4TR{W;~~W}09c=1n;E zXCqO2m0&c$9jk4B3@$FNvo682usRe&zMcNFvGw?EzdXlFo0cLbEwP!p|3i`7EOYt0 zC4@S~q*d+Wh6iP&ryXdHc>_~Z_73BFBn(>!FT-ty=Vufh)a7Hl4u~sYs31dT?#UeU znDCynDW}Hnwql&+dcRzU)JG!qe(G~bm)|Owt(* z!%5}D9JS%3u7q+bMiuk-|lZYQdrA+5n0oRQ! zQ8&|S)!W%j&nZ)+xfzD#TK%ez6HCU)WTUrFO-mnRiU^shn05(M^C?*(ts;ep|JG-5 zyura}DD2}7(j~58E-d>y_kTHs<`N@5}t1CoR&Y0XJxk~4URT@%eJa3%l!TGlqVeytQ| zPP@GnQ|ZNHZdh@1tuYKP-rEs(I6KK}@2?=d#PnT0(eHDHDBurk#mP6+S3h1qv3};J zj22*cu7e*oefq`8Jd(9&;;P}7+SZ=u8jh4chIV|z%bhm^e%l!|TyArzc*NlIPFo_0Cj<|2w6qVX41X_CXCYT1fpbtXdlG1T}A5v@q^- zI!}c;)Pu*LYYS(!E99(u%J@a#amTpc+|sx@0)WC&HnK5rR#N5)WjGnmDMP`x_zco5 z!xF9AI0`nU_8I0R=4?)vnVdukI12LAf_OX69nMt*jUsW+Y=#kLo{ORs|MK|qXL1eW z1vN1+EuN?bXgAL01PrvruP(*VFBPdyktlB!f?%5KRxQW2Ey9 z<80n47r$+SrS?Id|00?=YT<4Uf|~yqJkI^^sctFP|AmCRUDW~qUzFv&@4DgB$K9~7 z5dVSi=i0kN67m=O9uuViR}FC->Xtp6aM`WG`q`-5Z}P<_qd3T3d!oTzrx))4Ly!j- zKIT_vXJ;?#hl6E+=s}6 z{Fx(gC2gr?!l=tt1qM)uaH#)lFti}9r*0UvEB@R0vH^!vX|zEgvQ{54;_Ru4Ko9ix z1Ps#P@HA-=RFKkgJfF~K2rEoUu>fw~HVVaps&2ENx@5|-Moodf>Ui=TJHb8_cI}5bxk_@+^;OX zYxy>$RWC!!Tnm>4p^#8e5~x!7(wdy0QN&iQJl2K0%HBH3EQNL8C|l(@)-||SU`q-Q z-`j?z28N)Jkx)wPh|^n+=TIWG*4SV?6e_BD{iP6{m>H05O>ZB>g3RD@QDm>lU^g)Y z`x+wQ5!;w#U;gNN{9>v%{))<9C}>0yWI0g@#0UD01ptkC|9(Sn#N$LjKvHfp>Xj_B9{p;VJTIq}38D8;ATW5A{^^GhMqX z9WceeL-g~~zvn%OcFaphwA`FB!pt|v+=4d;^~}<2cA<9!f`QoVBpI^i%aJ(UM^#UF_!F}+7L#5fJ$bq|des1whah6@2k}_hE>~nlrDRp6 z=^Et9m$+GHA;|PFOfBO8jIaL&e_(L|0y?DbR8^T!*Yn!bmxZ^yj1)I@?8-PxN7XRo z+CI00nYCa{X&Je@S3({26hEPo(`THT4_9fAaWI%LXICH4qSxDD)*l3NX2ef6os|sP zM=D5ESJKmmJu8ay-rCVdBAb%HPna}9teEBFLW%J|HZ*6xm|<3@KslovEH6Qy4J2>1 z`Bm7p&KV6+XtpZYJ~e9})ixO;lc3HJ+$*b$-qeU%k@wAxHafbpy?T|4ESaTl$AM2q z09gCchO*$DRuS63S&6nowE`4xedInO0PcLbf;HSR%kCJZ_d()oi5*IR{8%8riB&mb^KIaeT*DzJdzge78mIO6jRaJTwKki z#^!;p512ejm$QLYgjeGv+5V4o*r4c)5lq!rKHQR$F0C$awH7ol&9Pet4 z&cLVNakv0!wdyM>G=3oAObWiykDk|Ha#1oR3I3s!QszKq-;X-$DR)JK8?{Ht;1+on zR(IW~3lppabYWFuHPZjkN)eC=H?Yzk6pW(F88XC>UCg#`S{%EDUQK>!zF|0!V@Z1@ zb9Hm=`HtgtRC}?CX}#*=p!b@AgLiysGa)ROaUJ`&a^t-`d6zdqd_&TTEnphzq+Y#y zetU7`OZVpXV=hoj(uxsTs-A=0t2-%x0OIFVWsTmo^VfDf)@W`#L)ISBibR7Ip-! zV!E|(in7|b8uV9{Xt31b5kc}2CGj3Ggs6$E3q*iXt{QzCk`L%wwONXg-b;uyaq)3* zAIpVYPN__k9`vmr$Hgnco}w-K7gB_9?bS1;dfM1#rj99eEx|C^HzKjmc7pzf0zOrG zCiZdLXb!B_N49D;hski-J%e}*M<|ftK9#j5tOA1j(6(#30P%`dQ<6|~gL2Ez&{+DN z=&1%l4y5z?oZ_1R4NC^V=eq;Rmm?S73P05j$lc@liT7h$lrL`ofHb9>bFT#}R*`QD#%9TUKDk}OGf$YG#6K8%G*wFFw&{`T!PIV~ok(FJ zgU#>|8NG7krp^`X~qSKPtvn<~5x2%n{v_w-l z^sI7aY9f)JM30p#=-+QPC8I3UUu+0mSW81Z69R}7b8jIS8xUK9uv&Gq+>FL1^P|`2 z8u^gg-BUON>40o*igkO3JN;3LxNw^&Hwz24@O&-V%c{-LS6uKm)k#MPXztWvjil#w z-`RA9{q-2^-7F(?`ofyM6qp9vW7k!OH!$PW=jHV#)5m8|RP`6g7pQ+ergrdFclXY( zp&5A0!l~VkwG>EA=U2DrA%;U@&^L=yHY1%7#tKl!&4V1h?8G|ncaHj0tEe^jTNiZ( zxZaS$;ZPIzHAo9+GN+ft_3Rsi8|nZo_ZsTj%2ro}&pxQc=0E%r#`mlfR3Z)^rgIIt zy18C&U-s^YZ#!qwEG4f6bBPQ-D=T7y_1ON%_3JbL=8kGCRx=k|;vJE}#Ou>n*Eql! zQC*ekK|2VTy_hFq9^t&5>ELC(T6!2q_Q1X3K z!zVJ<1@qPJvKRvzNpY9^d{#iVAg=7zRwXTr9%&}pf@J)nnRw-6*XyxgnwD=?;#tm(-Ak}h%Yf)q(!He>@5D*wM-$?xuZxieJ?};R3T&5)r zm7_9dk5QTmh=)a)JJ^O{E9=?aCZLi+ZbrnX+ExFTAgtgxfWpur6|<}+BTGDhFU*{U zy~bfCEXBf4wtl_sz71w>fIO*WK@;}xdz(E)SN-`}pLC+LsQ=g;>KB(FIqBEh)f|k{ z0PJo%1*I?yQ8O|$6Lx533m3BPLWPLnu~&l_z?_mPca8g$T%;do&-($oB+ESc^+9L) zN%9mQz*MSdcz{bR2kWM1&ET@@xn_5)<=$W?+i=`leI|8*Q}}(rd6^vanqYE z3rG}}#q{B7+xz0!XSG>uhBMk)^+Go9Z)WCRYKv!c;cpEGcY)^+U$)0?;UMqy(m-UU zvMa=bUOGA7!j;-zS$IPJ84}B*ei`mxfn~K7>RxEIgEo>4!?*Qz^2 zFQ&?>i~g84rGf(4ux1gKs=Y7UZ%rz)Cqs8x$6Xovfmv1w$&1o@k-@ma+CBwnRnsOR-=%)9uQ5gV55+$;wu>{p3=^Oh(AlE{GibfP z{~QT)kD|Eyms2|0f<#D%wh9!xQndO)B>9T$;Q))hP?3db6q@;_O(eXq@b(#C-ctTd zA;{agcgII)RmL8K*N4jf*bMGE8+?>27FNz;T`3Y?^*-;X^e9c@o|;sJA;_GjsrdX} z+}_c%=66P~>I2x$?j*|8A-Y!g>utUPubwMas*I+8&8|KQ6l=t2WVekE$tv`%wqPZC z^Xc%Z`t~K*J5rO>3sv{QkhcojUfm|R!uSAv?p_%jgZ^0qm`H58VUAQsaF#)1=`-+0 zqQK|nVrTNwm?Xny=c$8OhYggI%zvHXzHBIk`rOaAmanF_%-2G%gDmVqfRmK#mKn2% z;PZR$!*|?+WB+t={g!%ZT`TAh9jx@g3&;E(#gMP->R%pUPY0bV-*D1FCJBz>Z! zJh$2-86kDUSEwy@{&lpPyF{J1#=~?YXtf$97k4(@GNL87R!^-O;tA`GlfWe#resCHXTDGUQ@wj9f!zL4q&-NJS_`g+snvcbo9{THWSVQwx#Xm<8sl$hScexa5tCGK@ z%Op<@1YDE4@w3W`ZPypQGo{6rLExG?hh`%#8SD6`7LyLH{*Z^M918X(R5FcgfRzSs zLrKk1xjClS?o5m>%|^5Us;{hi>c=&|eHj@5#oC#M$f%v0+lmTz#^U$XdUCvaqT;RA z&>@yJa?}8};S5-VQg&OlSylCn3`dY)(^8c?>GC}x?WPg>Q3YCK?3>nk`zqe^<(!vp z)tJhnO0DK{ajnFp`&v_xd?U*$^@6wfqE4wmmCWIh?EQ8odPkLxK3Pq@26E;3?{F58 z_^Mh$9ew+R?Eo0>3MRbx!or1y1qN*v?%UWr7!5VVFRh9B-i4_NXiu;9@z+A^ofNGS zM2~>iJ4w*PS(0a}AV^mLZUy3F8@To5j|zG|b4vrA3+H@04nVdEcKsgw)6$o4{J`wz zzBEpjT~sLyGb@JOEKa&Szx^YKhxkVPO^=V2km)dbAB0}#+}@RlchICPaFe?+;P0ZTek$VY@W%g=`0Rwf{PNtbqbWdO z;_j!|F8t;_lyC!R^flJtg|jkgfmz?7y8&JD8M7+VlJ;Z0fCjslVZMPtk3a(~kXJ&f zC#VrcOCkGHBra7(*|rCA4ZqRg>*<(gnJf2*B1Y18={Kn5_s;NLuWA?Q_ARdybW1!_ zI39p2aX}73%{@E}c$L~Aecj%s%>i{0BJsWkBv@v=Y!6b0^8|FMVr=Vu$;8RTwrzXI+IQ})|J14a`=P6Qef>Rj z)#`^{YZ1r5VS)Yc0MUZc8%e3Km`KTU%6ai{7_)1zn5uGE>8SE>Dr#tO%4^zM8`+zB zsVO@MNSWE%fnEB&x@>SiS%9j0Axb9Il5kbi;s@E+W#qjGJchHIuNrf&bFUtb5t9`$ zhAH~Mh1bqqzn&Qhx=@uP;$4$>icHW)Aa9xYeQzTJVz~iPsB!d>2Tp8|J$^ri^iYnu zZ+kZe*$Gf0>Ykq;10n{D?s^Bdy7xTz-hKG$p4?GT+ztJ>HoaOpeQpWy?2DhF6)7l= zcaT@EtGL@74*!zAqPLL%;%05jA_iE1C%-&AST4Ot^b|xZ0{GQFf_tb13&^ zQUC|=1(Tdg<}}?aj`Qlzl=29X340QvqecGVq{1O7{)h)B9ts)_N0!RPV6b37l z`{c6YXnjasRGo5R=X&>-Q$UK>tpEiC@XEWN_dn5^eh?N=sEZBd2pi* zBA21o!F>taPC(ok#r|&Sq|4?ifqm5z!Bz&QGur^h{@&ytpSNe==rc1i2l)e#x!Yn? zWSAGmh0h82%)<3k!3j)2eNaR)F@Zr?um{F49@2-1OpOgIaF8gi?c|({*fk3M?Y<>q zmfaP|wYSSs&DtXX?kl0k?W&KXT!Axr6i5k3I7Bb7IHl&0Ki|#zr(vzjA zIwWQ?JeTtm(iL7Mk$boYOnmrU(lw^yixs2b>o{_2KIlANc`Wk=l9-fHhcP6ua3u=Ggm(bJ-ZQP8Ly_?*y#hFq*Y*{oH*BLY@6nmZi&& z(cT7!j6>ah;N&wC%19h=d!=7w6rzM7iq-Z{8b_ZOt3AbcZHW1Yew?u+j0lsAQ{0n4 zos&w;p{7$iJrb#~^+nN7jkkE44Yl=9nJa z<7tcQ0N1^NAc@eB^snotG1!bCA02mi z&p3CbGK_fi3v=iu3*PVC!x=D$8ZSfS2qCn0`UsE&$#`nGNX5FnUX>{oapOON`i{mi z*Emj;@aU$8G3@`jWR?aogzO7Uy`Ws45dCJiwy3ixbxEThc)|zg9QDmZS zMg#1ZMYvdqfA?qrb6#0Svu}lCympfMMAda$mW6uAMWbJ!0VBKP=;!&3vK|Nqhhn|z z?-~|iGA5fo%+q*V56w=#Ki&&=@|5zGUJ_rG!ETKFPdi8k6~z@4`3qE-eXDh4-#*p( zK6^B0C%RX4s`iCjYNUOn-NoG+cZO53tvGZi+Uo0I21FRxknLS-FN$(%9K=MzxEc|# zv8Ur|Q#$b72#**8{-S7T_ctqtFg?L0Wk}sTdy(+N53MM)10g zrMwZe1tT?6taUfv%?csxMqn1qpSRO|y|dxPk8Qy{&M-0O(!PiYZ3pz4q&w^cNPTbL zk(@!xGuo#QPJmx8&o@*mebMjQ6m^IPz20r%@bY|q*&=?rdOCSKzu6-a8tUcf)1=E? zIZpF*X6uqIqMq#tZCNk{#FoVHr+jricG? z(>j|-Obh)fuvd7SK92}u^FPGlHN&bNx)q6=eL@1-;U6Gx{PEHk0C zC$IK`A91JHoq=}*?2YCAjcFqXpPM5`BkM2N1Iaay?jmd4T1LMsV$OnsQ14nc;z0^K zVKl~4 zP3xQT5^(eyA4>Aj0M+2nGP@@zU)4q4j|fZ&l0_sd%lOl zRS%C8#>CGAl02*t@Ka<)t#=Gl<3D(7BDJeJI#}buXHq95Xh|3&h6(-xkCb#>jJXPY zNIaPJ1ic(jeEZP`rf;8cNjJAr7mc-$aR5i(=kT_f!muQ2*!z@tut^tmt&oJD4}A~v zLhy&vLo#i9Q@(|G-PvLu1O2_|&QS$}xsA+uMe+*4d;5%o?`# z`d>Y|HXL-KsiKb2T56&;G<^N3niK8VOlL;9|0PFL;NPzesooZ!^U%Ewfr2zHHF2>k z8N#hZIb+_(a|iZWs;6{bxZ?AcPr<@xvgk2C9O(VFlhwDKD9FJIg^PjRjiVb}&tl=f z0!* zY)a^Qe}g3Cj;UTRR<^M|evy=JWC_`f7`1yOl#C(hJ{7NaNhE#Z!}^Ow6+drU<3FNF z^yKF>#N$eiMYRbd+Ypyp?BT*ljWG~qRBg*foOWQ0xI)A}DmhQrQgxV1h0O`HhWG4P z1buq6pO`{+(;M~UU4f>z|6Z&>f2>^pTi9EGi1%sXbQ`BRk%}q%oN^3ssH4RoJTS_) zDN*ge%+*#EZo-uuOj+r^aS{P>9%^%0)?-NNibPXJay*LLqTrQki0a7qefnJ9oHz+C zb+Ir|qCy4|ns7b6+;~Mlzjv~eI^*f1>fvC`NG-x~5BWcpOOSiCK6`v^$CG{!oukJH zOyGu-jQn@4!6yse58{<+xm=cVO+Hou^x$w;IqsN#^h$d3td=Gt^LZkcRRqpj%ER~Q zbmB-BXi=g4Rc7U~ohaIa<7y_98auLd1d(!EAB37WMSF99q*wGM1pqnU+^ zm`|4@Ja8cuyJGMn0^gTV??LsfpRBp}{rk0x?#Yl($J(xGrja38)WWMtob^E|AUKJc zhMc(BK3itkjwkGQn*A4mc4^H1%y=*1Z<}G?zUHK1*8a$TI~jcFTXCYsCbN3_<-w>V zRogb=T##qza1QjTyUb?AQ0L&(wsDXF3lVv_!LXfXEGv}9uXTd*&+q6M!sTD+SqIgeJ#({A=jCj>dKTnJMQX~O%bm*O5sBJ1<1IL!i zsgvrmriEU+O_j?wFa|VB&KOL_7*pRNMWxZ8r4kGuwUEK3FXwJgT(6yuT+hs=867d} ztZREHdT}yaUf#)~>kwp>wZMrH(MS9iY7~2vwyI~w063&tT>k`u{Vq~C03tL?t&38$5rZTGjnh{ z)qKXRy(z_ne-8d#93W5{jWlv)K>y_tosk1UToAC{6Y%2cRk9fM8=1eQ&U90l{BCd> z72Q4OF~@a^>|9+%y;nmvO5FGhNkfA%j3`^IoLm~r%Z=&|c{bx;9sK!3r~gVGEd;58 zSYi3}I8unNmnFqVE4YMjLGRKANk)Tz<$SNWl+|V*Ia;_)74DWsV5=bFgj(oY|JYyR z)r$TW1s(2!{Ocr&`K_aaEg5Jv2nMN)sJ{2Qf%C zNhooY+oRnjdNVSwN@RnuVr>YQ(yAN$_L85nI{Wlp{FWv5Rr|nbX;?{)<9xx)6-I9j zE?u7HmBUItruAInMiS6R|7R+v%GT;PQay%I4Juid)n5Y>5g{7b;8|8#)-6mjlaVDe z5j1786By;6wd7-Mt53gRzFJ8zVO>0ADB9Te>ub6cF9u})ZY8*v0_%ksF6IF+c>{ayhg=D$r96v+jg0hpAHG6gf_8f<159-5Pb6=-<-!Cxe``nnYxv*h>6$85N!8dmhVrq2E1Vg=CFOFLC3 z8|Z`Rah=1^yTp-PN*ASEk7ntwk=g(qXfh3UZhNihd!``LJZxWgtfCm$S9HyZdZK3Y z@Oy0&`_nX&MJSpH@)Uy=6!*%}m`MNWVDOH4fLPzkrF1Y!5*}B`2;1B*0E$Ybtmz_V z?ap$0i+UKMNo7qouKApmKwYKkMU~D9^TC7#)g)bm>-eQD?TuknBbxHB%^{m18uO3> zDQT1*G@M5R1^erT9;nfICc3|)BXOPc@z@*b>qiWAIw1)|iZv$N#%I7j5`S1 zv$0Bxmwn#K1m5l)e16ULh(dbb#y?0{+7xNu-Y2L_+nV_k?3BD9g4)YIqWB|T)z ztA%WsAei!tA3Xoep|7dX`%YB6?JXfh#9C>aaZf>RcY-rez7L?OmGO6o2O40KZ`Bq zIUE@wN(H6NR1C(oFFuWqP6p9X(i+k$9jG{QIzr<4M=m#NjWCANTnV7^SPZqg4MIno+-d#a8x#`Rs%%(c<9ByPF#h4%gNST_e{uSM_n4ZtF?+qMs6X^k647k0>~Zjf zc~Q+%z{i5IM%%N_Z5r9Sn;QWCG}SeL`E6;*F7q#5{0bixsi|C3Tyko?DZ&p_VnthV zF8QhAh1mgrO!rQ|zcwx_)mZiQy-^&fL&doStiF5;)Liv}B@uVNsBxOh-zaQHONGV2 z6yjO_{T9u48wMzNAwdsb;ZxpbB_jyFyIVEEZzoHp|lk9x3loK+EuXoIcM-D;@G;OQ_ zpHKT1++QVTgMQ`+;TWi}MOQom04(Ju`B z^y^<6qap1zOGd%Ga!L{K=PV+GROJIupD3&%DzUcTS7{t~qclGI$Ph&9^q4uuV>M!d)gn{mdnNe@0D=w5!Z3U32FBzi zHhm&rTa>Q6p5&3SjZ~yC_%azPtMgep~H)^1WYC>2YjExg4V5sPm)|4{zQ5J|V&^0daqzeT*BYOwL=yUK58yUPD5 zpb#znx9}fuBt&v3A5}XtUZx0b)r~Y|Hx~y}=yA%*JZC~0bjD`QqJA9XRBCbS4{%|v zAnugYXJm70rwaDGdF@bnk=37P(l-`4W|FNjs*e89x1X22|Mr9WG1_Kz(%=b*CtLP7 z4n}XABtLLwKE7ifpaVtig^1xnQ@uKAbXR`AS&)}EIictI3(q-Bm2xcHllAbI<=b9i54L5ZysdlLJr!0wJR&31`;H;Z@D&euq-Ff2omut^^N6 zhL&=~llQa(Ai`{pm=HzH)LljfXxFgL4lZCC(>nQUOC62X$M*f)Q?2X{ynaAXado~g z|5@Owhpy{R^>_8c*oexoVdxVrl1wM^zxuw6l>B3}c;(BPh%`6B2U)NPvtha$)NwH5 zW@#UC!6aM}WtQ(Q)*g5l$Qf@MC6a+}9M6X#FE-$)fN0iJvmnDK2K^&tY(4M|z)OVA z^$GYsoB=Yox->>qq-veD>&xqJx!A0^P@_Q-Y~(R+dMOou4DTWe*}}4?Q|{qDd^p_9 zWI{_t7WbR+M3Py`^jwsP9mM;w9h4rF&z;23H?L7CT0!@XsmVet*mHNEL51Un^2rV; zfz-7c*#^j7K&;!>X^P^mAXVE}A4;1RCqX)7?}^SLlwm@t>#%u1&tkeu7xO4*@{Kqb z^+ZO1S_-`%htS5K-e^;_QvmGq%ZwfQ<(%>Iu#B?|@g%O@y_?F}+}(m40>6`M*5fYC8dh27 zxPZnU^14FtfRO@+cEnlQY5=kE1I+SsaGCDfhS`ct@;M#TY1Q~Vt>J{OgmM~Z6XJ2J zT2J3j_J-d#pov~(x5sCy+rVegT#I`+hNrk$%gIv$e~*QiJb(XRj=gEy`Fa}`u15O0 z!5_%x6RriSlg)^#gPdx>bPG%_Q>n~-mQdJwG&jB*nLKWOAzCu+ON`Of<{F}Olu(B2 z_t&vi*Wm>u2dK#()EnO7KS=5q;iFG%Z|tOiptz zKkGjpVZa5BGcuM@c|=Snx>al=LCB4L4<%S#4vfk72w;_i^5BN^Ta6|^it>u#Nrz+4 zD6ZA`o>iPMMC;(d(J+oi(;0o0wn@#!BI-w2CDZv;$bD-=+|9C9{dPXnbH);7W{PdG z-mvE5$d-9J(d6w@+uFyRDoSo5u3gH~az>s+uShBCzx`PfZ*Vvg2KTs&a)oD@563Z_ zo;ER*Bja1P?ph`X#&H(f%rFTCj|0{Lz2mWe)z6q$#X6{Jl?-FdgqWCD)%3@Ld?8z> z6+YIb6cCjZkSt6^Z$>7QlF2vZnq*puC;9K!vc9$Lyjqeq<@QoSqZf~}Y01OA&NQ$D zv?b|qa+KFTSVejP^j$qM9&m*!5DaO>$u~CCJl;I9f99u-6k>aBfFCt~`o+mSlDBH& zsS%Xf)LrBo4wpTKbwK0iFPMTqk;vCM&d61jDwLr(pTr3iyR$ql_R90}dGyTd{VYx` z@aOk>XQ?JgOf7C){#z&epavNutUe2;mX0@0%NG|Tg7=)kTWJRM;E{7M zkSa}&fIPJz-Y)Wo@)bcNC_FP;;Y3*%VrV7K9$)?}uHk&sjpSccn*jkq7473Mfqr%YRu$?{waoJDecQ0tKQ zYU!L*lux>f_zSi>c5FrHcuZz*-*SmKatRWL$%`lz2sY%DySwC4jo1v+l-TW!f$lZM zI})Ub zykP6Dcb5peIzlvybjJBS-Tu4xxIITP`1A?Y^w2s%x!5$y<*Rn_+aX+TAK?8DqKT^( z;bt$S`G4R^{(nw&OS%5HCBp5xe)|7)Spa=E44*#kheU)04t+n@-|dr8zc}_;s06ud zN#fAA?chbq?-Vx9N8}PImY#maLH61a59~RWL0{mmnHQb0B?K;|HTdJX;rQ5ZD+yGUO_s zWj37Mm4tMdUrXVliF9hyRdcK((KTb}QD z+}FG)3Hg9u80u|Z)mnF6e!o7h-fTmcsnStu3K|ZI_ZJ(&NjYV-WJpfmp3Ra~+i48> z&M{2-5Eb)t3ov*iuL$AmXx7D!^iNGDW@FdovvTT2#ZG?a&AUyb$E-q<6$>|7JUAKE zw#EjfVa!LTlyV?sK9hDHyYs-D1%KE@VWVbxXz97in;PwXdk#hj{7Z{@k9GL(F}V#) zVM+UFa01#KPtUh@tdv!Nb?h}Ij7rCU&Ej&Kug1gVGryDm6X0u|+#4BN@~#O20Q5XU zg%%I63>3F=UQ3w+nW(QbJhreMzl_}vN`t(w`1sfY^4~K=W0($3J<7+qo(P=#xf6j{ zMn;5;BOJscj(Q3|c59sqDS<)XyjXxHvP8Cvb|#;+abDpgSIn|3o>qm@fz*yfuV0jH z1g7C{``LQTZY{vvlT!XqQUUZS@9^PLT0`nEue&@-kLaQHc}0)%-mMuAAxCH)MH`xy zNvK4?Ow*+82)pPffFbMc|J+I>zfT?SmBlAZWveLa(~FfJZAe35S#Qo23xhoLeqXZa z2m(KgC-XmsZ^}M}tH4FU-Kl&7f*wB~Ctt7E#=-*BRmf{gMzQbudSstjo7ud7iVM-D zA$}$LHgOR|N3S?i^SC1ojz%%=;J3t&;=u6I$L(zQvL4WjhAxW7Cph;lfsmnG9HQyV z+jfjw=L(15lN(>%ST`<+Q$%QL6bt-0(xb4Dp%IWDqnQ%oBpdOUFJZ%bxPOjKy|T9c z3fcsG?yOB=un(E=4_((}u=A^+lr z);~Uq*l9NI1USMGakpN(?M284w35052PYy5X!M(g21*2jOZNPNfB&M}acK4dT49>ls zkekz}a;FkmQoS9%k(5anK6Yr9`2<5Ij-E2u@>NBynTxrb+&C5SFBpHfK(d~npkqbm zI3`6w;`|-~AwHh&N`y=(5+0|YWCjC4U-`;ze_ETdBchM(3vC>636rIf_pE}>VJ4%B z_RDP{pA^bt&osj#eQ~T@?=s{Fev=0(G$bNncf2Gr+4YB&$^Oal24l>2fTPLWq*0&< z2^p&wQi`Y>+YfOvlm- zv6~!p>C5v((gM5Dy;FUJQrUcA8zbN0Vvp-G!30_>J>ELkR|60Q^9=FOF5s??z-gCD zsp>rL>%?|Ly_Qlh1g1gncl$5ExuLs_gN3T(0o43np;t$}qX7RI?V&C2fh3{bTY*V> zX8FQuD`!2Fydq?^_D$qTN$)4R?S|P+iOF9g(U4=-Yt|s1!bAJ!ZvC5Xts?2)G<9#a z{X7;PE{iwmxoBd2y|uN|%tHBwIiL9fn$L#IkeTj?u^M>g&{(acj$UtRq|7sFH`9WK zJ$|6qfwj&V;zq-)$_X;rVp~cmu*nWA=OWXYY4m@xeKa zw#E>Sk(2&;^a9W$hS*-#Bn@fY7GN5b-{oD6+)P-+r^;4O-Rbt*QBC>Aj<@8nkxqDc z4h#_(?>ag>s@o2qSZw~1_pzy$;_zW8uAiiTtgf$bfc8z)ei|7z5?F4+`kFbPG^-ET zVH^Y@N5FQ%LVxUBUvJw-fP5lHoN@>b>rx!+Ezp7`)2TVrt;oN#^g7WCf20jvV;87Z z!fT?5NdSoiw!MpxoV_zca@zcn#@QsNzKyMr*|`5~S22eo1uC3ooB;kb=df5)sCD9`qo{vfnwfYCmt<}aOX2&Ey>7GI%e4i8d zajANAMm8sDNr?`3Miwl(`eJND>~@E)mdy#j-#ee~w>~h$i`D8MA(OWAkX(SM*%j$077UI+!+2vtQ zI@%vRsj>bEGg)%QTwybJ{_&>vjRW3oT)MmDTEXV#rnxnAMc)-doxib$M#aUlicxc5 zPxz;-U7~aiWXC@z1JJSZ1(${`ag=Vq4us-`^7Fa{zI?r3 z1~4q#c(;V7NcFRR3g zlL%)m#e|FA9mAT+Xq$4Ee2>Z?!`?n$b~Hv?6fA8EyjM(B;Wwvp8yrP3f>m`k<=FED zkp(M6^VyanQ8_Dh%eAjXp;qW0f1hJl@{2hmKs4X88IDD>yPUCO0L`G%p-IyYvtGo{ za|&1jrsd6tMNN3Ex~34z;xS9PryomtKfy4*^%^<4dIkURsyX5K~#3=h9PC(&kbEUw2(cN;bDB zI~6wrd-OECS25Fi_gdDKo&iglCRNFaSEbsdc9IHXewAq%xxk^U*>A&CZzAuS2GDV$ zx@P=l^vwU!^yACM>fuE6W5uxz*bxX?_uAU)B|4_vg+!|K2{~m(#N45Is@gn;NiZy( zdvr1^%@ra!ud-GrLItYJPr;xWmhK_%b|-}skwuwOjUFm1Hw66wV)u&CAEy9BOiHh_ z)+mtWpCL~hjlB&WYf~Pk0^6-mcW9t)JW#N0T)Jx^qy09l1k!jd+IvHZrv4I=cM$qs zJcI94%4+J+wGvHMOje@?tyR)|nBMBd#9=jA=fz-4%)|i;rp%)=Vn6$hdLZ(G{%n6(gFYqxWGUXG$qQO){!UK&om3 z>44WPV(>P69P>Dsf7_+PqcC&B&@3#FP+HC_hq7tH({AB=HSb@R5nLv@M)YPJ_a`hb z6QTaNX3uwX%oDm_fB<2Zjq5xD>tgPEuX??^btggYj`cfx{{k7b1aDMUa|}>YHervw z=-EH;K?gLTy=i3d5FNKB@MwtK&!9VT#jz6=z&#*IX<< zg#H4|DM4FuAMI0<;m&fth9P^vk9c296IV2psOfA5iPIB*P`ZIg$j2sXa`6NwrxoH< zpy34sX2K`q;&aq2s9XKI_AMfl;gK2DH=*F7X7=}m2}bkvk{*2dIemU#XQhR1zq*0S zgi|X4=49YFw)u3NIpX6$$2AV>1hzO&EI#3^4st1{CFxb#iEY?05!YIx%2A9)b)ir? z>os~7_Df|?^JlFM-(IjMUd_W{8|hI(1~b9PS!(LM5d0%t?q-C2W4Z}ui>qCyjv{;1 z>6_55OjXXLCzDGy`IhZk?6gC{=KOGlJkn1q4IeaJjhlDYl+porzHC51!pZIcpw+kQQD<5p#28v!n#gC2cehZt<_N}cn^^Nxol%Qc1Uv}P*} zD!wqCx^kV1#R&C^O<$`oT^G@G^(vb|j^%SBH-UR4iPOZ#C9$e1p>nfsG~32JcN zK+wk8#t!p7$e{cBK0si!Y4_tsL_R@5aPO);ShvWzg?AXhHbufDdoL0OA{ZC$2;7cqc}^=g;~mGa6L5C%)4*_)3q5VoCyi ycJIH=TguXK_|Vl) zRd=1=A&rB>0{hPZ(StF5msMvomsR3X^x@|=TU*FUt=``1hG-A}OL*%Jlj`MHnqx@U7I;FFlZvGfI6g^KEA z4|(mnhOfiv=npv%y@M=1ao)ZnYKZLtVCvdC z7Jmp|h{+I=)QEUGvT!^xb@bvUnuE4k!WWEQebBL6yMAUqH${3}l?K9prbHvu0Gqk* zZ+`K$ZlD9b!!`&2gQBCLydg?RB-V5Xpu_FMAu>@R4qR#0bx{m)9#t7ys_7(%Bxs^7 z{6!?T6r-AGCCB%uOg%6+csg##l5QhrE=yO>J~xgta0o4B?J`EBfGDoDmixw#|UTvgB8vP zxa>OK9MO`{qF&m&-Zym#O7pqZW=uiMLL=ulHjA%r(h^5WssYe>bFbT!7rtELcLlRM z7qXB|pp^vU$$%dQLv;|9s*0LG#zJ|YdLn4HOP@m7=%H1^qa32=$m7Q@ct$RQf8Qc; z8)+LpkhbrRPn@7S*bkp}+g>Mfta&Ed$-;Ey7{WOCJKgU0_VO_P!ivmI`S`%vYdtPL zDgfgq=<@Kw#`{g#1x#3LSX?G0nMq8f55^=3(vO5flLISwm?WbEa85z$9)tdN-x@W~ z>Hfg`cb~16y-yh2U)qq*-3Ujm3TOH_m^wcB2))qyl$KlRe9vG!cX;&_zR^A~snK&8_qeJrc8tQWljxnrkn<$9iR>FlQgUW}`uqt(8Pi9k zstqywcot^jj33>V?C9|qPpw3H_8R{LT?Zd4$!>XHDdzO?o^$m4%Iz8V&;XP2lIPny z?N8lsaiYWxXdV|_yv(^zbC_RXO}KwNSBOLFW|NS|BVxzJXwkb3@bn`JcMVKim8(2K zdmA1x3-|PcQ_4=RAala+mwT00juC?>)jR++jlV3{dQ0uwkqV7)ov|g4iIGiG-IGC` zlS?n4X41Mmk*ISF#4yg{XHvGjzpM$quMjvRtY+yUousn4p z(U&&FU-t)tq{G89zpk4nV6#F*B^s7Z^w@{fIAYQXi9r|~thANn4=4%|d~i4}xZ&h| zlYG@`FjBFvtl`^i1V#Brb6^m4J|@UfqG<1oQ6Oou$@EC^s!d138VefIrhfwUA5Ubh zbDk;_+|z4*KsX8b-G|6xauMjYcCK@VyH=eNXd*Y`tw@*R#)vHMVG@d3Z?XMFm5sU` z8}GO(&dWwx)TjNB2V@)1y%me|+50&lp=HpzD%wXW5&H@qKej)Ke*UXT!3)9oNU~q+ zUE4ZB-hA7Sb(Ub~vBkxo>%DNVK=qgEYYI>e?B;vmX&2eBij=a7P@y`jf31PS+ozV` zXP?geRPVZe&4E~Jot&SXr<6z2-e@|u4Y$Em$L|K1A#o-SWJmY&pYY9-Z==8CU)SS=2%$s>0iY~cZ2%PGoAK=Wbr!p zWaiKcz8^4#Cc|%5{xVT5e>LjamT*b}0q=Hj1o%I{>=8fRy9+%N)P=h{I~cva}g255qN|nV2M>bax0!V|BUqD@XOF?%cQzCrlL*TK3u4SCQJM0 zcVmYlcJFxeGS9T3KolV|2|Q&Y5+b7QB(Jr}eW%I6HOJMPhN%eUZ-(YUb%_jyq4HC@ zf0Z;}38U^*db0?QANmvciZJaI;q&tp=@bG*`XG51(cKix+A0|L#jV&-5PrK?jCqm6 zPJN#cOcQ``C6Ty(scjXtybs6O)sJNPsNT#)XQ8|lY%F|qgnwGE(^X#1H;||q7DgWj z$dssXPKyGM-cmpBe9~1#x`=x9W7eM6Qc6aQuDLxn^b@roM_=rY={)2SzvwV)-n#O! zZ0Y^eUW1N-N#SIVjZlpNzMdZb{K;Of`64*^pzEs}oY^2_(#tC^fGFynp%S@Qxt~vH zJcFcEVNAlb589hO3dzrDDF>s&RAk>?#>j2@4qonr@M%n0StiQH2no{PV8fIG=OdnC zZ{sdMdV>5Pju8nAgIRcGJ=0GwbiT#fOuK|*@AC#Y%4gUXHJH4QyEvwcdC^WJ%s~J_ zLeQd6EHE5~pLE_KA2-*TN1*+u-QTI8DDOW1jW5KU6w~lD*a|#xy$;@x92_qM%E!#B z1`|JFqsv#8;blozgv}L6`&!VIZA&uC%W+(o5FRE%D`?w6AkeYtq}V_nP=k63GDwU?JqLD^-rSnA$?{xs>abTpGR;cP) zQqSn0mHE>qG7HTpEe_heV^{K%#qWnJG!z3^Dg$oy0AN)#D)2xp-aLa((Nt>%ndX*u zMZap&>9ij`wNvivt(6yA6fgR$?PPA#MfaN6!lr33LzteYA;khT8nUNKaVFfpDJ>kH zazkrA1#=N$>I&JhQDQvI?>-c>)$o-bxDoR-&I6H||`{@p#PuIT1P z&lF^I&A%abBcJtK`n|u|>-Nn99Msy~?DczTH@!59C8t zdrgGp_Tyu>Ep?MG6Y9lJMkI{&K=<0w#y^`yPgbHGvjd3f9nBy#YsN03UrBEB0`rhz z-*F}F!*9Rw`8vmg11Rkxe6xp8aSTuCO zjj4i?i*0Kvq&{FoB$iLaJmakRwfWvj<}`<$l&!ARC7!L=Id@SSTqq^#(jNaZmfNNH z2#fGq1|UjO#BL4Vm58L#lrD-Nbc0bPKe?HIizR(`l11X-`S`E#M8h{L=C8i!gPK8gEbW0YiD8#L_ls0kP2P)USB>L|***ukdmG_SpU4y@^jx!v zj%anhi`V>=zEzo7+xe?FxboA#>pPs_O@TTP&JDQ3S zmKEcs9ODTzRdg4&CpK6t4l_Jr1*r9W3Yv-m*wbHaH>4uQ#`GR$KOj}E81Z@MWrZ3a zB*TfVg=vkNqkYpgR~lB-DN?dT!_u`BipEaR&Sf#gjPvFEA-Syn`3F8bGo1Y}hsh+* zqf)uM_z#jMS&@!OGEC2Axp3bJ!hT}Uwfgc^*-Ynnhf$ziV?hYXtMH!w1x_USWp@5j z3*1ELk%St2iw5~nB>KidNi{}1abvYFE3s&t^RIkpV>^(KVtQsMV*lM=KGzDDn3dT7 zd89(cF>7?4N5NYRu$}jmvGsJ3meY3v6B1N`zo2;&K^Iaq4_x)uZDq&ONO*I{&j;7x z8MMTd4^+}$Dz58Vct@waopj&?Fc|35LDov?UW7$k>Hl8SokyJvA#1Q4v{8SD!k$N46;)=GoV_I^{z?b2gSsoD3JxjAZh+u1di z+ok-ze5B!bC|2~LJbG>nBmT!&K}ePomT+}@1EUTHDZY)MXlv+S#3QE%;(@~}9JR1@ z*+u@VS%h83c--KV_i$xvtzr2L_Y}pynoWc2xen28x&3~UMVF4w>^CZM;gy7?onzoX zg-dzy?65skJ^%@l^jRToiK?2@sDBgj5rFVDVL(BakVy;#(Tzi zRra)^AUnF%wesjmUqK#2REjoh6OP!Gq`BZq_H&C)mYu}hCpXwP@DtlS-C-=J?3r%@ zsX009G<}ZN5Lk~eCiG~>KarB9TgJZd*_-SCTt2_$Dh~IUb`g#Upf^HXkKwGgf@H(!+NwdwtR=|>2PhY zrApC3hvPD=^?=DOYLYf9+$n0_gVD?xhfb4XJ3L11=y02smMR{VUbn@b5#!RS9J_W^ zn^tyoiVV2eym-#8kl^-+XvMT#*;WIOI1T$OLqydsv#)K%#0_7bMPXQeu?lOQF-rFl zv;y{nqGWjSG(`G=1ilnEsqknq`>Lz`o1~o#RcPQ+6f{!?Gr>cuJ#i~}T^Nt;5MYKL zB|29-Nl=Cb=Xd&oe~dCIrpey1qOa^lxtCy_7T9tU;YaQGC2h9}zKd-;h5o|JgFtr@ z4K6@te56lQY|pMl6(8QF*qNR{lV52Jmcl|YVmuk$FY!bcJBAJkUq^_}E_}g$A#PWi zqu+N$7dTH<%)4w{JS!@lnVXei{9w6`%4h8lzIVpKJ_OXIg+Jnz#LxA48sO&pH@h7LG6Oj!k<3V+SZRDJ#!K5aaM@+S-QM#fsmq-C)h^FEk7huEQ^aP)} zSWDk*qP&x9}Rq(T#Gb+D}^{cj9KWU+VCG?3~dbaSrSuBv)gb<&>9 zw?>PKE*gsmmn)}$@VjbdQ_@IW?1Wzp0E}Tumjeoj)q8FCd)#LAljv&CuJ&oe4eY6K zr6(J}S+>X>7WnE?Mf*u4uLcd7c`qxvp0x52nZ{(=?YdmD8&7LDfQ@_+& znczCly{V&}M^+&U=JNMY0Y%j)_;WV#eo4i7Zdv$_pzoL$Xd!ti7KcRqq(X zFzDkNIU`D~@0{-5fj=XRvW^o85B5VhkImfU_*Z05pXv0%gfy4PO9pd>TJGY{+Mqc7 zM&yrB91pZH*VH=K2{0&BgInQ-&8HT#ZxY$wQf;im-?()FR~^|LvJ7Q!L1#oEPs~^h zFM_4oQOoDa`o!urlC~bQbtqLUj;ziAIE=<=Xd*RcK}J8gOWu$C{H6eYVV4XVU+;49DIQ5Vo+B+fnqw)Ebk1zhxE`; z@qCO$Lnltrs6K{l*RQW0>q|YeMIlo+xpPY04zGqHC^Pr*=LhtRT<-w^PjlvH&*-8} zFUJ5vuEmmbSN47T#xI;da=W2WB&QF(-U{ltEHN)cu9JFPRTALXhchIvWQVN681(kC z}c(*2?F3 z6AZSH<3qaYQU-KxoJM99Z0sFuLk5 zC$hA#(nG%+mh5<2BJg=Sx`GwFMFi2DGn)~i2|(+477fr{9Ny32teyI?R+0V5pLw*b z8DfUqgYx25A}Iz}9$7Q6sToDVh^@i%B7dy1ltz_x4SAZlSsNT>rmbl}IVNOEa|Qpm zd2?2I%r_kkv}7$preRRR<5)E(SojJLYkEO!V9L`NkMmW@)wiTs1%C_HMo7jzR9*f% zBOPg`3mhJ3gX^Bx*|?kke{u2=Sjd+V3VDDl86w>mqi`E*ewH0f({rc$E>|y_E(>bi zXQL>xnK_JhVkr^(tKT^ls;alLedjFC^X=`mE2Q?AC8>r!PgkqS`Gv~BeMM;JEt;Iqk>^iiW36F-}9#nWl<)Ak=#R6$h>E*{dqF7v`5BabEi_ZZa4kc zuWrOSTl-yzsw~E=%bHA;IhIgwXavTRc(=TX@d>Z`AcqAG#l~aqFHcCG| zCQrr3nVxl{rPVpgL-tdHYY2A)#=|JQ-jJu>{WI-n&yCi_(ymy1l09p@lZ_qzNdC=xZjt4?96q#HeB%p_-Lr4G(qu1kXH zgv&`rcBQZl4&qZR&Z|MJM&y-`zAD?Ch<iDo0(1g8p&$xf6{e~tc8c6AjH-FVz%)2u$YI;K_YVL&CP&?2UUnY>_RZYgK7wh}!P(C(gSQU9{`rewI1QgBi z^+vKi?Vm~V6lvb>Yj+E3m~R!z=Ew2WGUyc@-lC5tHDZ!~c?F%7-~wI9C4@U9yGPe( zB@*t+j$PR+F!ri=cIYw1lmy6`5jn*|p@SX&yySa#JI4WRk?u4%OGC1zBuWA2mscC( zkL4K-5{G&8=EX;QRLK+a+0hu5C(7*kRN2*|De-)@p6%^Ko>f+E#DVk(YbDUU`Y4HQ zQ_fF9gwc*fh8nAUJdh%k&fNi1ugFE1GVB@VsrL4EpWUB|AnW7CAJDs=V*Cz4ze?0U zqp#9ou*c3eFWvx|cQyeW>MsWYn0;aeaF*<-3chx%_Lwlhuyg<>?U@9k(2Q1W;}XvkJxUS*3TSQ zA|&jV-u+V}=!E|_3zpL zTC`wRzUObvY5i{_u;RZ_tQh|dK)Kldo-bJI_J6ZBg!Y~@h5P+o&{R?RP5PhvwvNL7 zA5=3B_cD+`v+Y@Gdt=w2l4}*tIvjA?8gu3&>f7C00}-P{MS!lauWyk0w>I{+d|KPx zoPW2!>^OTlSzEsda&4c=+7-ykPWKbm@94NV?~R+U-!q4cxeq6v{ zzx^wDe*D;Xz%QS7IsI3j+!)Ab@dXz#1rOtrs!#yF68JNc@*&~!#wBAwNf?RVgFRu+ z5@lt){Rj?}OjA{HSb5eW)-pEejP+Ea){tLIyWHcuU2v*g55?$(PO!XRB-D3{IXCb` zD;ebr7jbE<%uRXiJ(CMYt&jJL1kbpBl(e$4qh+ptDKfh^9>d&qwKPlrZZpmn@B2Z- zuTXcer!JwNve+U`{vcwpw#_m|W|Z8|+<5SheZQq8RnQ3o^^wetDF)wm?q`bxb#=wi$K)rpKf_!X((?o>guuTV2XOOmd+H z{2u}XRY2b$;Kuhr4&slx62n!TcHHJtZ3wWIIk|Bo)%dYsN%V=L1spAqRK}xqEudhk zEjjKup@d7Eihkl!#R&s&H zeleKl(Qo1VuFliXV&(T4#sOFM1i##H!jV#3Bvw>pya2s;C+j(s^Eoh&TS3>!FFK0D zl$_fL671FsZElJ!8@_I7EhO1IAk17mk7OL7rtj+Vuix1s__?9YW*)M!QHPQBp0O#D z++tCliQQ~PzSR*u;@^33e zaJZC^*R+fr@lZEPQ3{w0>0IB!!V#bhK`&J=Q1CwtX%{EE; zhW(BbW%JJetK)>tcuOm978b`w;atMyc>;+JuE;ZbZC<+gkN{9UXcf^_bkf4(Bi`=eOX^dAw_ptUgUt4b_AuHrtP!_V}QP_k?0?N*qO=)0<5 zLE*Q>m6yV3bD%`BiT5fARb8^rrd|qVL>TB2Zf&-otxc5MID(uWXh2aNZa>}oIQtJ5 z2OKW71FUH)bhOA*!Ki5KWrywf=0Tj**Q*(A2dWF(O76%kCFUGe6TYktth=5{PoR}&8Q%{daoN$}6jh|L6h+?i{=(GmL= zSt5hzn!GWBw3xKK+7`4a#|SMn4%P;{lQ}00Pm<ig; z?KSkU?3R7J5)H0573z^UVmL0OMGGI?n1y57Ps%|$gzrv&#vT+P&4`P7g~&!Wmye#5 zzdYdb*c!TAd&hu5qQ*o)B3*oee*V55uCIR|UT8@ZYd+>~_-i^0oOJJDiTa{^j?3iI zxRny8{*@J8a&7Y#iFvuade!0zOwm3t!`@erl+L~qF->HXz3fZ5{ER!8ie5X7uC94cInYliA@v}vC ziM^#{XH|ocNDezZt0Ha*d(^*WqJ6a?8scD6!(QgA1?`gANW3b0Yvg5`!Tv?w@NA^? zp*m_4Npi9Lo}AFD)wGU4;y#GTz1#fx8UVa(bd71{4^-%wDa~$B>&(!gu>hhtb!15* z5nb2y#m>Bb#1q!qI3eRyStUhaW%!g8VB_$#1+#KYeXaA=j`?N`g7oq`k)Ch5BVPOe z-kQD?27|VCcDBxLgHZ4I5>vxI(lJn;ktA+U?}tw~9H1aicpagm_nOvwA=2cz|00-; z|JW@VGjK{1S7vof##~Dh*E^{MQULWWX6os5CgaI0cH#g1AggDB%zWd?U&TiEXN^b$ zO-ZPL$#X0EaPXR&q=`fHPy>yNCFXaD&2;g-DTo8RDtWSeB`hCON00AQH9l}xD;w(f z4eToI;l7vJOhwQ*`$1lju-|}tTYjy36Cg<;XhrRw4zasd)ZMiP?Tz zwwnQ;wi2%kvpLN_yy~Hrs#=k^Y;bH(pawhw5@Fie|RcLL?b1eXb=-!{e#B@*QIp^VR8^< za@s)4@>@}7((cW-;54;IahkXaae4BLCEg(4yXS+WvBS7I+_4)GlLF#ikbj`}Q$sZm zg6o1a2ej9S)Ct{ehI;y^?hPKJo;{eCmR^buo{A+S_`|-EhX(YLy4%-N7x1HI2YyZZ zQFhaC<+dhQ>uQD#^f*Dg_Df{L=k;*{PyFso!y(=Z9QsD|>*4j`Y-8(it|Cg5$wj+j z33-`lv8;-w)vtPl?m>=VSw3BRV)q;@H_L?po#%5>jzwN);=R(LVbCY027u?SM!7L) zmz5HWeHv&;wTDVVUu772~&v4rUsGD7rsRXc@>fg zFY`DThU@$j$SjlvQX&q9LW;v;Xubo|J+W+Vt6sVtMVjibc__C4NVl;gB%Tq>IIL}ey4~^vpxzl`gtEFE5?4UdK_|jkMb|8stk%rB0iM7E z#npwX!{?X7v86Yo6DXisxt9cTh%meeABCy*QsPM;*xANMq@^O~D;t^NdN^C!Pi1(d z*GPCZE$NELIcAY^?o}tPHZGdvq^ckjV`P0uujS*+6rp4{k#^QYGn1>2Q7N5;M|c2i zTUar1m8Xtw^*F!reUCjfGg3#>e-Th%DD^I1zWS>= zIuRAT*%ST>n)HkDQ8kMxwJm8B;8yT3wlQ5Th>7Gg!t|i-Ie`#dm9n0w#I`=zbw&4_ z8xmDG?9Y%6Vi^uXjuLD@d~L{IBmTPhUH~C7P}s#A{nf;FmtfXE>EH+kv?MSheX-mG zLKgQ`YaidSnXD|^gWXvY?1Jqw+MJUAl(h;~B*Dfe^cFO^6KpY4p1|X>I9Gw*Cu%1Xlwj+wYffL8`HrPm#j|? zy};mHew99;_$_@?JR3)CK|GrW)>0n23zEY_tlT45Q6F7^{5BZw^X?Z%F1}HA#wnT_ zF~#|oxRh4U#UgWS;crRt071IS5%?mgj+DeVc80(8g;o;ER>M(p*XHgSr?9eAD=lhO z?pyqbcTl;gs9&kxZ&|vC?$q#Ii9nK<-s9#;-lIRYO--qI9hoB^^cAiZU#^At61WfE zh&<@Q0)UjF%(tR0)wNoppe{Pl)>qA1_sEA6EUX{#7l6EAu*m5r&WuEGi?%L^uu-o0 zPBihJ^m-eOtx}7ZZ~%e(saYhnvg+v_G-3L z7gE1WrAQms;9k&R20X@!(aLU>1fEw0s6S&R^7!KZqKo(u6qsN{8w76(G!knD^*8qm zZZp1u-iLO+13;p~8f@gYy|5=5qxfr}@vJ$-6Uo_^wK5l~iuhEMR+s6c1jlW()0}@` z_+Q7sKe+$-*7nu%mIEyDIm-RV^PFYfcP-c@L|@+f9}x(S0D)OlMs1C920vgv^l`I- zFP)0|RKmY*>Ri26M5isPf_(PFpp2X%P>e`Z3q2Z6i_doR$n~$?h zVKnR6+&sAqDoIy8+P!t_|3QQ8)8JKGwpMXOn|2{ii>Q(-W=>6oo0+6y06?u`lro+ zLwWfy>dkX)i7|T*k9Bqa>=lHJMk>Na(z4z5h+&RRD)b<(k!(2Q3T`{i1r4q2Y$uRt z%WADB#o7Z2g@?Ty%&uB}Bg*31A7{e!6ZsvR4rBUIz4dp6Z<;3NjFZUx)bEo8A2i zL6EfY^L6yp1ZsfKHU(sydOpkRY!9S96@FJC*ho;OAeyW-bd(NQ5D8)kl2Sc z4}jVq!e@l)qM=OoL8vdrq*x!IH9luo54UMw;MT7i z!$bd^Og54`J(;g9=}_D$kn-Kv*hSd&*I1ZP5)H@(uQByQ#y=wybW9l%cN7%V;QjTW s-+O%o0xz&U2tutOI{&y+)^b;Q`8Vy)fgtdIJ;xwdE4V1IT5z!c1+qn<6#xJL literal 0 HcmV?d00001 diff --git a/charts/v1.22.0/azurefile-csi-driver/Chart.yaml b/charts/v1.22.0/azurefile-csi-driver/Chart.yaml new file mode 100644 index 0000000000..89b656c29c --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: v1.22.0 +description: Azure File Container Storage Interface (CSI) Storage Plugin +name: azurefile-csi-driver +version: v1.22.0 diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/NOTES.txt b/charts/v1.22.0/azurefile-csi-driver/templates/NOTES.txt new file mode 100644 index 0000000000..3fadd8ad36 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/NOTES.txt @@ -0,0 +1,5 @@ +The Azure File CSI Driver is getting deployed to your cluster. + +To check Azure File CSI Driver pods status, please run: + + kubectl --namespace={{ .Release.Namespace }} get pods --selector="release={{ .Release.Name }}" --watch diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/_helpers.tpl b/charts/v1.22.0/azurefile-csi-driver/templates/_helpers.tpl new file mode 100644 index 0000000000..b1bf4dc1b6 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/_helpers.tpl @@ -0,0 +1,49 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* Expand the name of the chart.*/}} +{{- define "azurefile.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "azurefile.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common selectors. +*/}} +{{- define "azurefile.selectorLabels" -}} +app.kubernetes.io/name: {{ template "azurefile.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Common labels. +*/}} +{{- define "azurefile.labels" -}} +{{- include "azurefile.selectorLabels" . }} +app.kubernetes.io/component: csi-driver +app.kubernetes.io/part-of: {{ template "azurefile.name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +helm.sh/chart: {{ template "azurefile.chart" . }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end -}} + + +{{/* pull secrets for containers */}} +{{- define "azurefile.pullSecrets" -}} +{{- if .Values.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/crd-csi-snapshot.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/crd-csi-snapshot.yaml new file mode 100644 index 0000000000..b0e29453c5 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/crd-csi-snapshot.yaml @@ -0,0 +1,661 @@ +{{- if .Values.snapshot.enabled -}} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: + - vs + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: + - vsclass + - vsclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: false + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: + - vsc + - vscs + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +{{- end -}} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-controller.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-controller.yaml new file mode 100644 index 0000000000..5dd4ce2a5e --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-controller.yaml @@ -0,0 +1,239 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Values.controller.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.controller.name }} + {{- include "azurefile.labels" . | nindent 4 }} +{{- with .Values.controller.labels }} +{{ . | toYaml | indent 4 }} +{{- end }} +{{- with .Values.controller.annotations }} + annotations: +{{ . | toYaml | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.controller.replicas }} + selector: + matchLabels: + {{- include "azurefile.selectorLabels" . | nindent 6 }} + app: {{ .Values.controller.name }} + template: + metadata: + labels: + {{- include "azurefile.labels" . | nindent 8 }} + app: {{ .Values.controller.name }} +{{- with .Values.controller.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.controller.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + spec: + hostNetwork: {{ .Values.controller.hostNetwork }} + serviceAccountName: {{ .Values.serviceAccount.controller }} + nodeSelector: + kubernetes.io/os: linux +{{- with .Values.controller.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- if .Values.controller.runOnMaster}} + node-role.kubernetes.io/master: "" + {{- end}} + {{- if .Values.controller.runOnControlPlane}} + node-role.kubernetes.io/control-plane: "" + {{- end}} + priorityClassName: system-cluster-critical +{{- with .Values.controller.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.controller.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + containers: + - name: csi-provisioner +{{- if hasPrefix "/" .Values.image.csiProvisioner.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" +{{- else }} + image: "{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" +{{- end }} + args: + - "-v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + - "--timeout=300s" + - "--extra-create-metadata=true" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiProvisioner.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: {{- toYaml .Values.controller.resources.csiProvisioner | nindent 12 }} + - name: csi-attacher +{{- if hasPrefix "/" .Values.image.csiAttacher.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiAttacher.repository }}:{{ .Values.image.csiAttacher.tag }}" +{{- else }} + image: "{{ .Values.image.csiAttacher.repository }}:{{ .Values.image.csiAttacher.tag }}" +{{- end }} + args: + - "-v=2" + - "-csi-address=$(ADDRESS)" + - "-timeout=120s" + - "-leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiAttacher.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: {{- toYaml .Values.controller.resources.csiAttacher | nindent 12 }} + - name: csi-snapshotter +{{- if hasPrefix "/" .Values.snapshot.image.csiSnapshotter.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.snapshot.image.csiSnapshotter.repository }}:{{ .Values.snapshot.image.csiSnapshotter.tag }}" +{{- else }} + image: "{{ .Values.snapshot.image.csiSnapshotter.repository }}:{{ .Values.snapshot.image.csiSnapshotter.tag }}" +{{- end }} + args: + - "-csi-address=$(ADDRESS)" + - "-leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + - "-v=2" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.csiSnapshotter | nindent 12 }} + - name: csi-resizer +{{- if hasPrefix "/" .Values.image.csiResizer.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- else }} + image: "{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- end }} + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - "--leader-election-namespace={{ .Release.Namespace }}" + - '-handle-volume-inuse-error=false' + - '-timeout=120s' + - '-feature-gates=RecoverVolumeExpansionFailure=true' + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: {{ .Values.image.csiResizer.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.csiResizer | nindent 12 }} + - name: liveness-probe +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port={{ .Values.controller.livenessProbe.healthPort }} + - --v=2 + imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.livenessProbe | nindent 12 }} + - name: azurefile +{{- if hasPrefix "/" .Values.image.azurefile.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- else }} + image: "{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- end }} + args: + - "--v={{ .Values.controller.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:{{ .Values.controller.metricsPort }}" + - "--kubeconfig={{ .Values.controller.kubeconfig }}" + - "--drivername={{ .Values.driver.name }}" + - "--cloud-config-secret-name={{ .Values.controller.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.controller.cloudConfigSecretNamespace }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--allow-empty-cloud-config={{ .Values.controller.allowEmptyCloudConfig }}" + ports: + - containerPort: {{ .Values.controller.livenessProbe.healthPort }} + name: healthz + protocol: TCP + - containerPort: {{ .Values.controller.metricsPort }} + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + {{- if ne .Values.driver.httpsProxy "" }} + - name: HTTPS_PROXY + value: {{ .Values.driver.httpsProxy }} + {{- end }} + {{- if ne .Values.driver.httpProxy "" }} + - name: HTTP_PROXY + value: {{ .Values.driver.httpProxy }} + {{- end }} + - name: AZURE_GO_SDK_LOG_LEVEL + value: {{ .Values.driver.azureGoSDKLogLevel }} + imagePullPolicy: {{ .Values.image.azurefile.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + - name: ssl-pki + mountPath: /etc/pki/ca-trust/extracted + readOnly: true + {{- end }} + resources: {{- toYaml .Values.controller.resources.azurefile | nindent 12 }} + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + - name: ssl-pki + hostPath: + path: /etc/pki/ca-trust/extracted + {{- end }} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-driver.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-driver.yaml new file mode 100644 index 0000000000..e1facb056a --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-driver.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.driver.name }} + labels: + {{- include "azurefile.labels" . | nindent 4 }} + annotations: + csiDriver: "{{ .Values.image.azurefile.tag }}" + snapshot: "{{ .Values.snapshot.image.csiSnapshotter.tag }}" +spec: + attachRequired: {{ .Values.controller.attachRequired }} + podInfoOnMount: true + volumeLifecycleModes: + - Persistent + - Ephemeral + fsGroupPolicy: ReadWriteOnceWithFSType diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-node-windows.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-node-windows.yaml new file mode 100644 index 0000000000..813859aee0 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-node-windows.yaml @@ -0,0 +1,222 @@ +{{- if .Values.windows.enabled}} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .Values.windows.dsName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.windows.dsName }} + {{- include "azurefile.labels" . | nindent 4 }} +{{- with .Values.windows.labels }} +{{ . | toYaml | indent 4 }} +{{- end }} +{{- with .Values.windows.annotations }} + annotations: +{{ . | toYaml | indent 4 }} +{{- end }} +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: {{ .Values.node.maxUnavailable }} + type: RollingUpdate + selector: + matchLabels: + app: {{ .Values.windows.dsName }} + {{- include "azurefile.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ .Values.windows.dsName }} + {{- include "azurefile.labels" . | nindent 8 }} +{{- with .Values.windows.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.windows.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ .Values.serviceAccount.node }} +{{- with .Values.windows.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} + nodeSelector: + kubernetes.io/os: windows +{{- with .Values.windows.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + affinity: +{{- with .Values.windows.affinity }} +{{ toYaml . | indent 8 }} +{{- end }} + nodeAffinity: +{{ toYaml .Values.windows.nodeAffinity | indent 10 }} + priorityClassName: system-node-critical + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + containers: + - name: liveness-probe + volumeMounts: + - mountPath: C:\csi + name: plugin-dir +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - "--csi-address=$(CSI_ENDPOINT)" + - "--probe-timeout=3s" + - "--health-port={{ .Values.node.livenessProbe.healthPort }}" + - "--v=2" + env: + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} + resources: {{- toYaml .Values.windows.resources.livenessProbe | nindent 12 }} + - name: node-driver-registrar +{{- if hasPrefix "/" .Values.image.nodeDriverRegistrar.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- else }} + image: "{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- end }} + args: + - "--csi-address=$(CSI_ENDPOINT)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + - "--v=2" + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 60 + timeoutSeconds: 30 + env: + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\\var\\lib\\kubelet\\plugins\\{{ .Values.driver.name }}\\csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: {{ .Values.image.nodeDriverRegistrar.pullPolicy }} + volumeMounts: + - name: kubelet-dir + mountPath: "C:\\var\\lib\\kubelet" + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + resources: {{- toYaml .Values.windows.resources.nodeDriverRegistrar | nindent 12 }} + - name: azurefile +{{- if hasPrefix "/" .Values.image.azurefile.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- else }} + image: "{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- end }} + args: + - "--v={{ .Values.node.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}" + - "--kubeconfig={{ .Values.windows.kubeconfig }}" + - "--drivername={{ .Values.driver.name }}" + - "--cloud-config-secret-name={{ .Values.node.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.node.cloudConfigSecretNamespace }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--allow-empty-cloud-config={{ .Values.node.allowEmptyCloudConfig }}" + - "--enable-get-volume-stats={{ .Values.feature.enableGetVolumeStats }}" + - "--allow-inline-volume-key-access-with-identity={{ .Values.node.allowInlineVolumeKeyAccessWithIdentity }}" + ports: + - containerPort: {{ .Values.node.livenessProbe.healthPort }} + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path-windows + optional: true + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + {{- if ne .Values.driver.httpsProxy "" }} + - name: HTTPS_PROXY + value: {{ .Values.driver.httpsProxy }} + {{- end }} + {{- if ne .Values.driver.httpProxy "" }} + - name: HTTP_PROXY + value: {{ .Values.driver.httpProxy }} + {{- end }} + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AZURE_GO_SDK_LOG_LEVEL + value: {{ .Values.driver.azureGoSDKLogLevel }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: kubelet-dir + mountPath: "C:\\var\\lib\\kubelet" + - name: plugin-dir + mountPath: C:\csi + - name: azure-config + mountPath: C:\k + - name: csi-proxy-fs-pipe-v1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-smb-pipe-v1 + mountPath: \\.\pipe\csi-proxy-smb-v1 + # these paths are still included for compatibility, they're used + # only if the node has still the beta version of the CSI proxy + - name: csi-proxy-fs-pipe-v1beta1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1 + - name: csi-proxy-smb-pipe-v1beta1 + mountPath: \\.\pipe\csi-proxy-smb-v1beta1 + resources: {{- toYaml .Values.windows.resources.azurefile | nindent 12 }} + volumes: + - name: csi-proxy-fs-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-smb-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-smb-v1 + # these paths are still included for compatibility, they're used + # only if the node has still the beta version of the CSI proxy + - name: csi-proxy-fs-pipe-v1beta1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1beta1 + - name: csi-proxy-smb-pipe-v1beta1 + hostPath: + path: \\.\pipe\csi-proxy-smb-v1beta1 + - name: registration-dir + hostPath: + path: {{ .Values.windows.kubelet }}\plugins_registry\ + type: Directory + - name: kubelet-dir + hostPath: + path: {{ .Values.windows.kubelet }}\ + type: Directory + - name: plugin-dir + hostPath: + path: {{ .Values.windows.kubelet }}\plugins\{{ .Values.driver.name }}\ + type: DirectoryOrCreate + - name: azure-config + hostPath: + path: C:\k + type: Directory +{{- end -}} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-node.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-node.yaml new file mode 100644 index 0000000000..60a746eb5d --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/csi-azurefile-node.yaml @@ -0,0 +1,217 @@ +{{- if .Values.linux.enabled}} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .Values.linux.dsName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.linux.dsName }} + {{- include "azurefile.labels" . | nindent 4 }} +{{- with .Values.linux.labels }} +{{ . | toYaml | indent 4 }} +{{- end }} +{{- with .Values.linux.annotations }} + annotations: +{{ . | toYaml | indent 4 }} +{{- end }} +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: {{ .Values.node.maxUnavailable }} + type: RollingUpdate + selector: + matchLabels: + app: {{ .Values.linux.dsName }} + {{- include "azurefile.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ .Values.linux.dsName }} + {{- include "azurefile.labels" . | nindent 8 }} +{{- with .Values.linux.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.linux.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + spec: + hostNetwork: true + dnsPolicy: {{ .Values.linux.dnsPolicy }} + serviceAccountName: {{ .Values.serviceAccount.node }} + nodeSelector: + kubernetes.io/os: linux +{{- with .Values.linux.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + affinity: +{{- with .Values.linux.affinity }} +{{ toYaml . | indent 8 }} +{{- end }} + nodeAffinity: +{{ toYaml .Values.linux.nodeAffinity | indent 10 }} + priorityClassName: system-node-critical +{{- with .Values.linux.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + containers: + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port={{ .Values.node.livenessProbe.healthPort }} + - --v=2 + imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} + resources: {{- toYaml .Values.linux.resources.livenessProbe | nindent 12 }} + - name: node-driver-registrar +{{- if hasPrefix "/" .Values.image.nodeDriverRegistrar.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- else }} + image: "{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- end }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }}/csi.sock + imagePullPolicy: {{ .Values.image.nodeDriverRegistrar.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: {{- toYaml .Values.linux.resources.nodeDriverRegistrar | nindent 12 }} + - name: azurefile +{{- if hasPrefix "/" .Values.image.azurefile.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- else }} + image: "{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- end }} + args: + - "--v={{ .Values.node.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}" + - "--kubeconfig={{ .Values.linux.kubeconfig }}" + - "--drivername={{ .Values.driver.name }}" + - "--cloud-config-secret-name={{ .Values.node.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.node.cloudConfigSecretNamespace }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--allow-empty-cloud-config={{ .Values.node.allowEmptyCloudConfig }}" + - "--enable-get-volume-stats={{ .Values.feature.enableGetVolumeStats }}" + - "--mount-permissions={{ .Values.linux.mountPermissions }}" + - "--allow-inline-volume-key-access-with-identity={{ .Values.node.allowInlineVolumeKeyAccessWithIdentity }}" + ports: + - containerPort: {{ .Values.node.livenessProbe.healthPort }} + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + {{- if ne .Values.driver.httpsProxy "" }} + - name: HTTPS_PROXY + value: {{ .Values.driver.httpsProxy }} + {{- end }} + {{- if ne .Values.driver.httpProxy "" }} + - name: HTTP_PROXY + value: {{ .Values.driver.httpProxy }} + {{- end }} + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AZURE_GO_SDK_LOG_LEVEL + value: {{ .Values.driver.azureGoSDKLogLevel }} + imagePullPolicy: {{ .Values.image.azurefile.pullPolicy }} + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: {{ .Values.linux.kubelet }}/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /dev + name: device-dir + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + - name: ssl-pki + mountPath: /etc/pki/ca-trust/extracted + readOnly: true + {{- end }} + resources: {{- toYaml .Values.linux.resources.azurefile | nindent 12 }} + volumes: + - hostPath: + path: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }} + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: {{ .Values.linux.kubelet }}/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: {{ .Values.linux.kubelet }}/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred + - hostPath: + path: /dev + type: Directory + name: device-dir + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + - name: ssl-pki + hostPath: + path: /etc/pki/ca-trust/extracted + {{- end }} +{{- end -}} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/csi-snapshot-controller.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/csi-snapshot-controller.yaml new file mode 100644 index 0000000000..b1f7eff92c --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/csi-snapshot-controller.yaml @@ -0,0 +1,65 @@ +{{- if .Values.snapshot.enabled -}} +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Values.snapshot.snapshotController.name}} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.snapshot.snapshotController.name}} + {{- include "azurefile.labels" . | nindent 4 }} +{{- with .Values.snapshot.snapshotController.labels }} +{{ . | toYaml | indent 4 }} +{{- end }} +{{- with .Values.snapshot.snapshotController.annotations }} + annotations: +{{ . | toYaml | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.snapshot.snapshotController.replicas }} + selector: + matchLabels: + app: {{ .Values.snapshot.snapshotController.name}} + {{- include "azurefile.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ .Values.snapshot.snapshotController.name}} + {{- include "azurefile.labels" . | nindent 8 }} +{{- with .Values.snapshot.snapshotController.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.snapshot.snapshotController.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ .Values.serviceAccount.snapshotController }} + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical +{{- with .Values.controller.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.controller.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + containers: + - name: {{ .Values.snapshot.snapshotController.name}} +{{- if hasPrefix "/" .Values.snapshot.image.csiSnapshotController.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.snapshot.image.csiSnapshotController.repository }}:{{ .Values.snapshot.image.csiSnapshotController.tag }}" +{{- else }} + image: "{{ .Values.snapshot.image.csiSnapshotController.repository }}:{{ .Values.snapshot.image.csiSnapshotController.tag }}" +{{- end }} + args: + - "--v=2" + - "--leader-election=true" + - "--leader-election-namespace={{ .Release.Namespace }}" + resources: {{- toYaml .Values.snapshot.snapshotController.resources | nindent 12 }} + imagePullPolicy: {{ .Values.snapshot.image.csiSnapshotController.pullPolicy }} +{{- end -}} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-azurefile-controller.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-azurefile-controller.yaml new file mode 100644 index 0000000000..09923c06a5 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-azurefile-controller.yaml @@ -0,0 +1,207 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-provisioner-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-provisioner-binding + labels: + {{- include "azurefile.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-attacher-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-attacher-binding + labels: + {{- include "azurefile.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-snapshotter-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-snapshotter-binding + labels: + {{- include "azurefile.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-resizer-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-resizer-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-controller-secret-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-controller-secret-binding + labels: + {{- include "azurefile.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-controller-secret-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml new file mode 100644 index 0000000000..4e1fbcde94 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-azurefile-node.yaml @@ -0,0 +1,29 @@ +{{- if .Values.rbac.create -}} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-binding + labels: + {{- include "azurefile.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-node-secret-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-snapshot-controller.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-snapshot-controller.yaml new file mode 100644 index 0000000000..0cff1ff01f --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/rbac-csi-snapshot-controller.yaml @@ -0,0 +1,80 @@ +{{- if and .Values.snapshot.enabled .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-binding + labels: + {{- include "azurefile.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.snapshotController }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-leaderelection-role + labels: + {{- include "azurefile.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-leaderelection-binding + labels: + {{- include "azurefile.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.snapshotController }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-snapshot-controller-leaderelection-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-azurefile-controller.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-azurefile-controller.yaml new file mode 100644 index 0000000000..66e0726acb --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-azurefile-controller.yaml @@ -0,0 +1,9 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "azurefile.labels" . | nindent 4 }} +{{- end -}} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-azurefile-node.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-azurefile-node.yaml new file mode 100644 index 0000000000..697b8db390 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-azurefile-node.yaml @@ -0,0 +1,9 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "azurefile.labels" . | nindent 4 }} +{{- end -}} diff --git a/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-snapshot-controller.yaml b/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-snapshot-controller.yaml new file mode 100644 index 0000000000..e77ef8f991 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/templates/serviceaccount-csi-snapshot-controller.yaml @@ -0,0 +1,9 @@ +{{- if and .Values.snapshot.enabled .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.snapshotController }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "azurefile.labels" . | nindent 4 }} +{{- end -}} diff --git a/charts/v1.22.0/azurefile-csi-driver/values.yaml b/charts/v1.22.0/azurefile-csi-driver/values.yaml new file mode 100644 index 0000000000..ffaad432c0 --- /dev/null +++ b/charts/v1.22.0/azurefile-csi-driver/values.yaml @@ -0,0 +1,254 @@ +image: + baseRepo: mcr.microsoft.com + azurefile: + repository: /oss/kubernetes-csi/azurefile-csi + tag: v1.22.0 + pullPolicy: IfNotPresent + csiProvisioner: + repository: /oss/kubernetes-csi/csi-provisioner + tag: v3.2.0 + pullPolicy: IfNotPresent + csiAttacher: + repository: /oss/kubernetes-csi/csi-attacher + tag: v3.5.0 + pullPolicy: IfNotPresent + csiResizer: + repository: /oss/kubernetes-csi/csi-resizer + tag: v1.5.0 + pullPolicy: IfNotPresent + livenessProbe: + repository: /oss/kubernetes-csi/livenessprobe + tag: v2.7.0 + pullPolicy: IfNotPresent + nodeDriverRegistrar: + repository: /oss/kubernetes-csi/csi-node-driver-registrar + tag: v2.5.1 + pullPolicy: IfNotPresent + +## Reference to one or more secrets to be used when pulling images +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# - name: myRegistryKeySecretName + +# -- Custom labels to add into metadata +customLabels: {} + # k8s-app: azurefile-csi-driver + +serviceAccount: + create: true # When true, service accounts will be created for you. Set to false if you want to use your own. + controller: csi-azurefile-controller-sa # Name of Service Account to be created or used + node: csi-azurefile-node-sa # Name of Service Account to be created or used + snapshotController: csi-snapshot-controller-sa # Name of Service Account to be created or used + +rbac: + create: true + name: azurefile + +controller: + name: csi-azurefile-controller + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: kube-system + allowEmptyCloudConfig: true + replicas: 2 + hostNetwork: true # this setting could be disabled if controller does not depend on MSI setting + metricsPort: 29614 + livenessProbe: + healthPort: 29612 + runOnMaster: false + runOnControlPlane: false + attachRequired: false + logLevel: 5 + labels: {} + annotations: {} + podLabels: {} + podAnnotations: {} + resources: + csiProvisioner: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + csiAttacher: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + csiResizer: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + csiSnapshotter: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + livenessProbe: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + azurefile: + limits: + cpu: 1 + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + kubeconfig: "" + affinity: {} + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + +node: + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: kube-system + allowEmptyCloudConfig: true + allowInlineVolumeKeyAccessWithIdentity: false + metricsPort: 29615 + livenessProbe: + healthPort: 29613 + logLevel: 5 + +snapshot: + enabled: false + image: + csiSnapshotter: + repository: /oss/kubernetes-csi/csi-snapshotter + tag: v5.0.1 + pullPolicy: IfNotPresent + csiSnapshotController: + repository: /oss/kubernetes-csi/snapshot-controller + tag: v5.0.1 + pullPolicy: IfNotPresent + snapshotController: + name: csi-snapshot-controller + replicas: 2 + labels: {} + annotations: {} + podLabels: {} + podAnnotations: {} + resources: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + +feature: + enableGetVolumeStats: true + +driver: + name: file.csi.azure.com + customUserAgent: "" + userAgentSuffix: "OSS-helm" + azureGoSDKLogLevel: "" # available values: ""(no logs), DEBUG, INFO, WARNING, ERROR + httpsProxy: "" + httpProxy: "" + +linux: + enabled: true + dsName: csi-azurefile-node # daemonset name + dnsPolicy: Default # available values: Default, ClusterFirst, ClusterFirstWithHostNet, None + kubelet: /var/lib/kubelet + kubeconfig: "" + distro: debian # available values: debian, fedora + mountPermissions: 0777 + labels: {} + annotations: {} + podLabels: {} + podAnnotations: {} + resources: + livenessProbe: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + nodeDriverRegistrar: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + azurefile: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + tolerations: + - operator: "Exists" + nodeSelector: {} + affinity: {} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + +windows: + enabled: true + dsName: csi-azurefile-node-win # daemonset name + kubelet: 'C:\var\lib\kubelet' + kubeconfig: "" + labels: {} + annotations: {} + podLabels: {} + podAnnotations: {} + resources: + livenessProbe: + limits: + memory: 150Mi + requests: + cpu: 10m + memory: 40Mi + nodeDriverRegistrar: + limits: + memory: 150Mi + requests: + cpu: 30m + memory: 40Mi + azurefile: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 40Mi + tolerations: + - key: "node.kubernetes.io/os" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: {} + affinity: {} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet diff --git a/deploy/csi-azurefile-controller.yaml b/deploy/csi-azurefile-controller.yaml index 2fe7d407ff..896c5f4b31 100644 --- a/deploy/csi-azurefile-controller.yaml +++ b/deploy/csi-azurefile-controller.yaml @@ -129,7 +129,7 @@ spec: cpu: 10m memory: 20Mi - name: azurefile - image: mcr.microsoft.com/k8s/csi/azurefile-csi:latest + image: mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.22.0 imagePullPolicy: IfNotPresent args: - "--v=5" diff --git a/deploy/csi-azurefile-node-windows.yaml b/deploy/csi-azurefile-node-windows.yaml index 062aa14894..80704ea0f0 100644 --- a/deploy/csi-azurefile-node-windows.yaml +++ b/deploy/csi-azurefile-node-windows.yaml @@ -91,7 +91,7 @@ spec: cpu: 30m memory: 40Mi - name: azurefile - image: mcr.microsoft.com/k8s/csi/azurefile-csi:latest + image: mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.22.0 imagePullPolicy: IfNotPresent args: - --v=5 diff --git a/deploy/csi-azurefile-node.yaml b/deploy/csi-azurefile-node.yaml index 8d1c01678a..bf1138e93b 100644 --- a/deploy/csi-azurefile-node.yaml +++ b/deploy/csi-azurefile-node.yaml @@ -82,7 +82,7 @@ spec: cpu: 10m memory: 20Mi - name: azurefile - image: mcr.microsoft.com/k8s/csi/azurefile-csi:latest + image: mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.22.0 imagePullPolicy: IfNotPresent args: - "--v=5" diff --git a/deploy/v1.22.0/crd-csi-snapshot.yaml b/deploy/v1.22.0/crd-csi-snapshot.yaml new file mode 100644 index 0000000000..18d97e6b7c --- /dev/null +++ b/deploy/v1.22.0/crd-csi-snapshot.yaml @@ -0,0 +1,659 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: + - vs + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: + - vsclass + - vsclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: false + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: + - vsc + - vscs + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/deploy/v1.22.0/csi-azurefile-controller.yaml b/deploy/v1.22.0/csi-azurefile-controller.yaml new file mode 100644 index 0000000000..896c5f4b31 --- /dev/null +++ b/deploy/v1.22.0/csi-azurefile-controller.yaml @@ -0,0 +1,180 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-azurefile-controller + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: csi-azurefile-controller + template: + metadata: + labels: + app: csi-azurefile-controller + spec: + hostNetwork: true # only required for MSI enabled cluster + serviceAccountName: csi-azurefile-controller-sa + nodeSelector: + kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node + priorityClassName: system-cluster-critical + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: csi-provisioner + image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v3.2.0 + args: + - "-v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace=kube-system" + - "--timeout=300s" + - "--extra-create-metadata=true" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-attacher + image: mcr.microsoft.com/oss/kubernetes-csi/csi-attacher:v3.5.0 + args: + - "-v=2" + - "-csi-address=$(ADDRESS)" + - "-timeout=120s" + - "--leader-election" + - "--leader-election-namespace=kube-system" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-snapshotter + image: mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v5.0.1 + args: + - "-v=2" + - "-csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace=kube-system" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-resizer + image: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v1.5.0 + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "--leader-election" + - "--leader-election-namespace=kube-system" + - '-handle-volume-inuse-error=false' + - '-feature-gates=RecoverVolumeExpansionFailure=true' + - '-timeout=120s' + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.7.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29612 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: azurefile + image: mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.22.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:29614" + - "--user-agent-suffix=OSS-kubectl" + ports: + - containerPort: 29612 + name: healthz + protocol: TCP + - containerPort: 29614 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate diff --git a/deploy/v1.22.0/csi-azurefile-driver.yaml b/deploy/v1.22.0/csi-azurefile-driver.yaml new file mode 100644 index 0000000000..3190ff2423 --- /dev/null +++ b/deploy/v1.22.0/csi-azurefile-driver.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: file.csi.azure.com + annotations: + csiDriver: v1.20.0 + snapshot: v5.0.1 +spec: + attachRequired: false + podInfoOnMount: true + volumeLifecycleModes: + - Persistent + - Ephemeral + fsGroupPolicy: ReadWriteOnceWithFSType diff --git a/deploy/v1.22.0/csi-azurefile-node-windows.yaml b/deploy/v1.22.0/csi-azurefile-node-windows.yaml new file mode 100644 index 0000000000..80704ea0f0 --- /dev/null +++ b/deploy/v1.22.0/csi-azurefile-node-windows.yaml @@ -0,0 +1,181 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azurefile-node-win + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-azurefile-node-win + template: + metadata: + labels: + app: csi-azurefile-node-win + spec: + serviceAccountName: csi-azurefile-node-sa + tolerations: + - key: "node.kubernetes.io/os" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: + kubernetes.io/os: windows + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + priorityClassName: system-node-critical + containers: + - name: liveness-probe + volumeMounts: + - mountPath: C:\csi + name: plugin-dir + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.7.0 + args: + - --csi-address=$(CSI_ENDPOINT) + - --probe-timeout=3s + - --health-port=29613 + - --v=2 + env: + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + resources: + limits: + memory: 150Mi + requests: + cpu: 10m + memory: 40Mi + - name: node-driver-registrar + image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.5.1 + args: + - --v=2 + - --csi-address=$(CSI_ENDPOINT) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 60 + timeoutSeconds: 30 + env: + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\\var\\lib\\kubelet\\plugins\\file.csi.azure.com\\csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: "C:\\var\\lib\\kubelet" + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + resources: + limits: + memory: 150Mi + requests: + cpu: 30m + memory: 40Mi + - name: azurefile + image: mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.22.0 + imagePullPolicy: IfNotPresent + args: + - --v=5 + - --endpoint=$(CSI_ENDPOINT) + - --nodeid=$(KUBE_NODE_NAME) + - --kubeconfig=C:\\k\\config + - --metrics-address=0.0.0.0:29615 + ports: + - containerPort: 29613 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path-windows + optional: true + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: "C:\\var\\lib\\kubelet" + - name: plugin-dir + mountPath: C:\csi + - name: azure-config + mountPath: C:\k + - name: csi-proxy-fs-pipe-v1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-smb-pipe-v1 + mountPath: \\.\pipe\csi-proxy-smb-v1 + # these paths are still included for compatibility, they're used + # only if the node has still the beta version of the CSI proxy + - name: csi-proxy-fs-pipe-v1beta1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1 + - name: csi-proxy-smb-pipe-v1beta1 + mountPath: \\.\pipe\csi-proxy-smb-v1beta1 + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 40Mi + volumes: + - name: csi-proxy-fs-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-smb-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-smb-v1 + # these paths are still included for compatibility, they're used + # only if the node has still the beta version of the CSI proxy + - name: csi-proxy-fs-pipe-v1beta1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1beta1 + - name: csi-proxy-smb-pipe-v1beta1 + hostPath: + path: \\.\pipe\csi-proxy-smb-v1beta1 + - name: registration-dir + hostPath: + path: C:\var\lib\kubelet\plugins_registry\ + type: Directory + - name: kubelet-dir + hostPath: + path: C:\var\lib\kubelet\ + type: Directory + - name: plugin-dir + hostPath: + path: C:\var\lib\kubelet\plugins\file.csi.azure.com\ + type: DirectoryOrCreate + - name: azure-config + hostPath: + path: C:\k + type: DirectoryOrCreate diff --git a/deploy/v1.22.0/csi-azurefile-node.yaml b/deploy/v1.22.0/csi-azurefile-node.yaml new file mode 100644 index 0000000000..bf1138e93b --- /dev/null +++ b/deploy/v1.22.0/csi-azurefile-node.yaml @@ -0,0 +1,157 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azurefile-node + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-azurefile-node + template: + metadata: + labels: + app: csi-azurefile-node + spec: + hostNetwork: true + dnsPolicy: Default + serviceAccountName: csi-azurefile-node-sa + nodeSelector: + kubernetes.io/os: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + priorityClassName: system-node-critical + tolerations: + - operator: "Exists" + containers: + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.7.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29613 + - --v=2 + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: node-driver-registrar + image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.5.1 + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/file.csi.azure.com/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: azurefile + image: mcr.microsoft.com/oss/kubernetes-csi/azurefile-csi:v1.22.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:29615" + ports: + - containerPort: 29613 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /dev + name: device-dir + resources: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/file.csi.azure.com + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred + - hostPath: + path: /dev + type: Directory + name: device-dir +--- diff --git a/deploy/v1.22.0/csi-snapshot-controller.yaml b/deploy/v1.22.0/csi-snapshot-controller.yaml new file mode 100644 index 0000000000..79c7483bb5 --- /dev/null +++ b/deploy/v1.22.0/csi-snapshot-controller.yaml @@ -0,0 +1,46 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-snapshot-controller + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: csi-snapshot-controller + template: + metadata: + labels: + app: csi-snapshot-controller + spec: + serviceAccountName: csi-snapshot-controller-sa + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Equal" + value: "true" + effect: "NoSchedule" + containers: + - name: csi-snapshot-controller + image: mcr.microsoft.com/oss/kubernetes-csi/snapshot-controller:v5.0.1 + args: + - "--v=2" + - "--leader-election=true" + - "--leader-election-namespace=kube-system" + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi diff --git a/deploy/v1.22.0/rbac-csi-azurefile-controller.yaml b/deploy/v1.22.0/rbac-csi-azurefile-controller.yaml new file mode 100644 index 0000000000..b9348c8cd5 --- /dev/null +++ b/deploy/v1.22.0/rbac-csi-azurefile-controller.yaml @@ -0,0 +1,194 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azurefile-controller-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azurefile-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azurefile-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azurefile-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azurefile-csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azurefile-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-controller-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-controller-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azurefile-controller-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/v1.22.0/rbac-csi-azurefile-node.yaml b/deploy/v1.22.0/rbac-csi-azurefile-node.yaml new file mode 100644 index 0000000000..3752f36aa4 --- /dev/null +++ b/deploy/v1.22.0/rbac-csi-azurefile-node.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azurefile-node-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-node-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azurefile-node-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azurefile-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azurefile-node-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/v1.22.0/rbac-csi-snapshot-controller.yaml b/deploy/v1.22.0/rbac-csi-snapshot-controller.yaml new file mode 100644 index 0000000000..03af765424 --- /dev/null +++ b/deploy/v1.22.0/rbac-csi-snapshot-controller.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-snapshot-controller-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-binding +subjects: + - kind: ServiceAccount + name: csi-snapshot-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-leaderelection-role +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-leaderelection-binding +subjects: + - kind: ServiceAccount + name: csi-snapshot-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-snapshot-controller-leaderelection-role + apiGroup: rbac.authorization.k8s.io diff --git a/docs/install-azurefile-csi-driver.md b/docs/install-azurefile-csi-driver.md index 9fca89dcc6..a835c4a99a 100644 --- a/docs/install-azurefile-csi-driver.md +++ b/docs/install-azurefile-csi-driver.md @@ -1,6 +1,6 @@ ## Install Azure File CSI driver on a Kubernetes cluster - [install CSI driver master version](./install-csi-driver-master.md)(only for testing purpose) + - [install v1.22.0 CSI driver](./install-csi-driver-v1.22.0.md) - [install v1.21.0 CSI driver](./install-csi-driver-v1.21.0.md) - [install v1.20.0 CSI driver](./install-csi-driver-v1.20.0.md) - - [install v1.19.0 CSI driver](./install-csi-driver-v1.19.0.md) diff --git a/docs/install-csi-driver-v1.22.0.md b/docs/install-csi-driver-v1.22.0.md new file mode 100644 index 0000000000..156429db69 --- /dev/null +++ b/docs/install-csi-driver-v1.22.0.md @@ -0,0 +1,45 @@ +## Install azurefile CSI driver v1.22.0 version on a Kubernetes cluster +If you have already installed Helm, you can also use it to install this driver. Please check [Installation with Helm](../charts/README.md). + +### Install by kubectl + - Option#1. remote install +```console +curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/v1.22.0/deploy/install-driver.sh | bash -s v1.22.0 -- +``` + + - Option#2. local install +```console +git clone https://github.com/kubernetes-sigs/azurefile-csi-driver.git +cd azurefile-csi-driver +git checkout v1.22.0 +./deploy/install-driver.sh v1.22.0 local +``` + + - check pods status: +```console +kubectl -n kube-system get pod -o wide --watch -l app=csi-azurefile-controller +kubectl -n kube-system get pod -o wide --watch -l app=csi-azurefile-node +``` + +example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +csi-azurefile-controller-56bfddd689-dh5tk 6/6 Running 0 35s 10.240.0.19 k8s-agentpool-22533604-0 +csi-azurefile-node-cvgbs 3/3 Running 0 7m4s 10.240.0.35 k8s-agentpool-22533604-1 +csi-azurefile-node-dr4s4 3/3 Running 0 7m4s 10.240.0.4 k8s-agentpool-22533604-0 +``` + +### clean up CSI driver + - Option#1. remote uninstall +```console +curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/v1.22.0/deploy/uninstall-driver.sh | bash -s -- +``` + + - Option#2. local uninstall +```console +git clone https://github.com/kubernetes-sigs/azurefile-csi-driver.git +cd azurefile-csi-driver +git checkout v1.22.0 +./deploy/install-driver.sh v1.22.0 local +```