diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..91e20a6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,72 @@ +# OSX leaves these everywhere on SMB shares +._* + +# OSX trash +.DS_Store + +# Eclipse files +.classpath +.project +.settings/** + +# Files generated by JetBrains IDEs, e.g. IntelliJ IDEA +.idea/ +*.iml + +# Vscode files +.vscode + +# This is where the result of the go build goes +/output*/ +/_output*/ +/_output +/bin + +# Emacs save files +*~ +\#*\# +.\#* + +# Vim-related files +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist + +# cscope-related files +cscope.* + +# Go test binaries +*.test + +# JUnit test output from ginkgo e2e tests +/junit*.xml + +# Mercurial files +**/.hg +**/.hg* + +# Vagrant +.vagrant + +.tags* + +# Test artifacts produced by Jenkins jobs +/_artifacts/ + +# Go dependencies installed on Jenkins +/_gopath/ + +# direnv .envrc files +.envrc + +# This file used by some vendor repos (e.g. github.com/go-openapi/...) to store secret variables and should not be ignored +!\.drone\.sec + +# Godeps or dep workspace +/Godeps/_workspace + +/bazel-* +*.pyc +profile.cov \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8342e6a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +# Copyright 2023. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:20.04 + +ARG binary=./bin/webdavplugin +COPY ${binary} /webdavplugin + +RUN apt update && apt install -y davfs2=1.5.5-1 + +ENTRYPOINT ["/webdavplugin"] \ No newline at end of file diff --git a/cmd/webdav/main.go b/cmd/webdav/main.go new file mode 100644 index 0000000..4d919bb --- /dev/null +++ b/cmd/webdav/main.go @@ -0,0 +1,55 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + + "github.com/sys-liqian/csi-driver-webdav/pkg/webdav" + "k8s.io/klog/v2" +) + +var ( + endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") + nodeID = flag.String("nodeid", "", "node id") + mountPermissions = flag.Uint64("mount-permissions", 0, "mounted folder permissions") + driverName = flag.String("drivername", "", "name of the driver") + workingMountDir = flag.String("working-mount-dir", "/tmp", "working directory for provisioner to mount davfs shares temporarily") + defaultOnDeletePolicy = flag.String("default-ondelete-policy", "", "default policy for deleting subdirectory when deleting a volume") +) + +func main() { + klog.InitFlags(nil) + _ = flag.Set("logtostderr", "true") + flag.Parse() + if *nodeID == "" { + klog.Warning("nodeid is empty") + } + + driverOptions := webdav.DriverOpt{ + Name: *driverName, + NodeID: *nodeID, + Endpoint: *endpoint, + MountPermissions: *mountPermissions, + WorkingMountDir: *workingMountDir, + DefaultOnDeletePolicy: *defaultOnDeletePolicy, + } + d := webdav.NewDriver(&driverOptions) + d.Run() + os.Exit(0) +} diff --git a/deploy/csi-webdav-controller.yaml b/deploy/csi-webdav-controller.yaml new file mode 100644 index 0000000..f88227a --- /dev/null +++ b/deploy/csi-webdav-controller.yaml @@ -0,0 +1,125 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-webdav-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: csi-webdav-controller + template: + metadata: + labels: + app: csi-webdav-controller + spec: + hostNetwork: true # controller also needs to mount webdav to create dir + dnsPolicy: ClusterFirstWithHostNet # available values: Default, ClusterFirstWithHostNet, ClusterFirst + serviceAccountName: webdav-csi-sa + nodeSelector: + kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node + priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.2 + imagePullPolicy: IfNotPresent + args: + - "-v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace=kube-system" + - "--extra-create-metadata=true" + - "--timeout=1200s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + imagePullPolicy: IfNotPresent + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29652 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: webdav + image: localhost:5000/webdavplugin:v0.0.1 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + capabilities: + add: ["ALL"] + allowPrivilegeEscalation: true + args: + - "-v=5" + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + ports: + - containerPort: 29652 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + volumeMounts: + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: socket-dir + emptyDir: {} \ No newline at end of file diff --git a/deploy/csi-webdav-driverinfo.yaml b/deploy/csi-webdav-driverinfo.yaml new file mode 100644 index 0000000..16407db --- /dev/null +++ b/deploy/csi-webdav-driverinfo.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: webdav.csi.io +spec: + attachRequired: false + volumeLifecycleModes: + - Persistent \ No newline at end of file diff --git a/deploy/csi-webdav-node.yaml b/deploy/csi-webdav-node.yaml new file mode 100644 index 0000000..8578874 --- /dev/null +++ b/deploy/csi-webdav-node.yaml @@ -0,0 +1,138 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-webdav-node + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-webdav-node + template: + metadata: + labels: + app: csi-webdav-node + spec: + hostNetwork: true # original webdav connection would be broken without hostNetwork setting + dnsPolicy: ClusterFirstWithHostNet # available values: Default, ClusterFirstWithHostNet, ClusterFirst + serviceAccountName: webdav-csi-sa + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: "Exists" + containers: + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + imagePullPolicy: IfNotPresent + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29653 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1 + imagePullPolicy: IfNotPresent + args: + - --v=2 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi-webdavplugin/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: webdav + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: localhost:5000/webdavplugin:v0.0.1 + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + ports: + - containerPort: 29653 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + resources: + limits: + memory: 300Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-webdavplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir \ No newline at end of file diff --git a/deploy/csi-webdav-rbac.yaml b/deploy/csi-webdav-rbac.yaml new file mode 100644 index 0000000..ef9f2fc --- /dev/null +++ b/deploy/csi-webdav-rbac.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: webdav-csi-sa + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: webdav-csi-cr +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: webdav-csi-crb +subjects: + - kind: ServiceAccount + name: webdav-csi-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: webdav-csi-cr + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/examples/csi-webdav-dynamic-pvc.yaml b/examples/csi-webdav-dynamic-pvc.yaml new file mode 100644 index 0000000..cb53f9f --- /dev/null +++ b/examples/csi-webdav-dynamic-pvc.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-webdav-dynamic +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: webdav-sc \ No newline at end of file diff --git a/examples/csi-webdav-pod.yaml b/examples/csi-webdav-pod.yaml new file mode 100644 index 0000000..c9d5855 --- /dev/null +++ b/examples/csi-webdav-pod.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx:latest + imagePullPolicy: IfNotPresent + volumeMounts: + - name: pvc-webdav-dynamic + mountPath: /var/www/html + volumes: + - name: pvc-webdav-dynamic + persistentVolumeClaim: + claimName: pvc-webdav-dynamic \ No newline at end of file diff --git a/examples/csi-webdav-secret.yaml b/examples/csi-webdav-secret.yaml new file mode 100644 index 0000000..19bd0ae --- /dev/null +++ b/examples/csi-webdav-secret.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: webdav-secrect +type: Opaque +data: + username: YWRtaW4= + password: YWRtaW4= \ No newline at end of file diff --git a/examples/csi-webdav-storageclass.yaml b/examples/csi-webdav-storageclass.yaml new file mode 100644 index 0000000..9709aba --- /dev/null +++ b/examples/csi-webdav-storageclass.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: webdav-sc +provisioner: webdav.csi.io +parameters: + share: http://172.25.16.2/remote.php/dav/files/admin + csi.storage.k8s.io/provisioner-secret-name: "webdav-secrect" + csi.storage.k8s.io/provisioner-secret-namespace: "default" + csi.storage.k8s.io/node-publish-secret-name: "webdav-secrect" + csi.storage.k8s.io/node-publish-secret-namespace: "default" +reclaimPolicy: Delete +volumeBindingMode: Immediate +mountOptions: \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2502d0b --- /dev/null +++ b/go.mod @@ -0,0 +1,24 @@ +module github.com/sys-liqian/csi-driver-webdav + +go 1.21.5 + +require ( + github.com/container-storage-interface/spec v1.9.0 + github.com/golang/protobuf v1.5.3 + github.com/moby/sys/mountinfo v0.6.2 + golang.org/x/sys v0.14.0 + google.golang.org/grpc v1.59.0 + k8s.io/klog/v2 v2.110.1 + k8s.io/mount-utils v0.28.4 + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 + sigs.k8s.io/yaml v1.4.0 +) + +require ( + github.com/go-logr/logr v1.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9f3abe0 --- /dev/null +++ b/go.sum @@ -0,0 +1,54 @@ +github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= +github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/mount-utils v0.28.4 h1:5GOZLm2dXi2fr+MKY8hS6kdV5reXrZBiK7848O5MVD0= +k8s.io/mount-utils v0.28.4/go.mod h1:ceMAZ+Nzlk8zOwN205YXXGJRGmf1o0/XIwsKnG44p0I= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/webdav/controller.go b/pkg/webdav/controller.go new file mode 100644 index 0000000..7897f11 --- /dev/null +++ b/pkg/webdav/controller.go @@ -0,0 +1,247 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webdav + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/klog/v2" + + "github.com/sys-liqian/csi-driver-webdav/pkg/webdav/mount" +) + +type ControllerServer struct { + *Driver + mounter mount.Interface +} + +func NewControllerServer(d *Driver, mounter mount.Interface) *ControllerServer { + return &ControllerServer{ + Driver: d, + mounter: mounter, + } +} + +// CreateVolume implements csi.ControllerServer. +func (c *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + name := req.GetName() + if len(name) == 0 { + return nil, status.Error(codes.InvalidArgument, "CreateVolume name must be provided") + } + if err := isValidVolumeCapabilities(req.GetVolumeCapabilities()); err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + mountPermissions := c.Driver.mountPermissions + parameters := req.GetParameters() + if parameters == nil { + parameters = make(map[string]string) + } + for k, v := range parameters { + switch strings.ToLower(k) { + case webdavSharePath, pvcNameKey, pvcNamespaceKey, pvNameKey: + case mountPermissionsField: + if v != "" { + var err error + if mountPermissions, err = strconv.ParseUint(v, 8, 32); err != nil { + return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("invalid mountPermissions %s in storage class", v)) + } + } + default: + return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("invalid parameter %q in storage class", k)) + } + } + + targetPath := c.workingMountDir + sourcePath := req.Parameters[webdavSharePath] + notMnt, err := c.mounter.IsLikelyNotMountPoint(targetPath) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(targetPath, 0750); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + notMnt = true + } else { + return nil, status.Error(codes.Internal, err.Error()) + } + } + if !notMnt { + return nil, status.Errorf(codes.Internal, fmt.Sprintf("target path %s is alredy mounted", targetPath)) + } + + stdin := []string{req.GetSecrets()[secretUsernameKey], req.GetSecrets()[secretPasswordKey]} + if err := c.mounter.MountSensitiveWithStdin(sourcePath, targetPath, fstype, nil, nil, stdin); err != nil { + return nil, status.Errorf(codes.Internal, fmt.Sprintf("mount failed: %v", err.Error())) + } + + defer func() { + if err = umount(c.mounter, targetPath); err != nil { + klog.Warningf("failed to unmount src webdav server after snapshot volume copy: %v", err) + } + }() + + internalVolumePath := filepath.Join(targetPath, req.Name) + if err = os.Mkdir(internalVolumePath, 0777); err != nil && !os.IsExist(err) { + return nil, status.Errorf(codes.Internal, "failed to make subdirectory: %v", err.Error()) + } + + if mountPermissions > 0 { + // Reset directory permissions because of umask problems + if err = os.Chmod(internalVolumePath, os.FileMode(mountPermissions)); err != nil { + klog.Warningf("failed to chmod subdirectory: %v", err.Error()) + } + } + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: MakeVolumeId(sourcePath, req.Name), + CapacityBytes: 0, // by setting it to zero, Provisioner will use PVC requested size as PV size + VolumeContext: nil, + ContentSource: req.GetVolumeContentSource(), + }, + }, nil + +} + +// DeleteVolume implements csi.ControllerServer. +func (c *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + volumeID := req.GetVolumeId() + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "volume id is empty") + } + sourcePath, subDir, err := ParseVolumeId(volumeID) + if err != nil { + // An invalid ID should be treated as doesn't exist + klog.Warningf("failed to parse volume for volume id %v deletion: %v", volumeID, err) + return &csi.DeleteVolumeResponse{}, nil + } + + stdin := []string{req.GetSecrets()[secretUsernameKey], req.GetSecrets()[secretPasswordKey]} + targetPath := c.workingMountDir + if err := c.mounter.MountSensitiveWithStdin(sourcePath, targetPath, fstype, nil, nil, stdin); err != nil { + return nil, status.Errorf(codes.Internal, fmt.Sprintf("mount failed: %v", err.Error())) + } + + defer func() { + if err = umount(c.mounter, targetPath); err != nil { + klog.Warningf("failed to unmount src webdav server after snapshot volume copy: %v", err) + } + }() + + internalVolumePath := filepath.Join(targetPath, subDir) + klog.V(2).Infof("Removing subdirectory at %v", internalVolumePath) + if err = os.RemoveAll(internalVolumePath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to delete subdirectory: %v", err.Error()) + } + + return &csi.DeleteVolumeResponse{}, nil +} + +// ValidateVolumeCapabilities implements csi.ControllerServer. +func (c *ControllerServer) ValidateVolumeCapabilities(_ context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + if len(req.GetVolumeId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + if err := isValidVolumeCapabilities(req.GetVolumeCapabilities()); err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + return &csi.ValidateVolumeCapabilitiesResponse{ + Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ + VolumeCapabilities: req.GetVolumeCapabilities(), + }, + Message: "", + }, nil +} + +// ControllerGetCapabilities implements csi.ControllerServer. +func (c *ControllerServer) ControllerGetCapabilities(context.Context, *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { + return &csi.ControllerGetCapabilitiesResponse{ + Capabilities: c.Driver.cscap, + }, nil +} + +// ControllerExpandVolume implements csi.ControllerServer. +func (*ControllerServer) ControllerExpandVolume(context.Context, *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ControllerGetVolume implements csi.ControllerServer. +func (*ControllerServer) ControllerGetVolume(context.Context, *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ControllerModifyVolume implements csi.ControllerServer. +func (*ControllerServer) ControllerModifyVolume(context.Context, *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ControllerPublishVolume implements csi.ControllerServer. +func (*ControllerServer) ControllerPublishVolume(context.Context, *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ControllerUnpublishVolume implements csi.ControllerServer. +func (*ControllerServer) ControllerUnpublishVolume(context.Context, *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// CreateSnapshot implements csi.ControllerServer. +func (*ControllerServer) CreateSnapshot(context.Context, *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// DeleteSnapshot implements csi.ControllerServer. +func (*ControllerServer) DeleteSnapshot(context.Context, *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// GetCapacity implements csi.ControllerServer. +func (*ControllerServer) GetCapacity(context.Context, *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ListSnapshots implements csi.ControllerServer. +func (*ControllerServer) ListSnapshots(context.Context, *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ListVolumes implements csi.ControllerServer. +func (*ControllerServer) ListVolumes(context.Context, *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// isValidVolumeCapabilities validates the given VolumeCapability array is valid +func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) error { + if len(volCaps) == 0 { + return fmt.Errorf("volume capabilities missing in request") + } + for _, c := range volCaps { + if c.GetBlock() != nil { + return fmt.Errorf("block volume capability not supported") + } + } + return nil +} diff --git a/pkg/webdav/driver.go b/pkg/webdav/driver.go new file mode 100644 index 0000000..0bb7b9c --- /dev/null +++ b/pkg/webdav/driver.go @@ -0,0 +1,128 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webdav + +import ( + "runtime" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/sys-liqian/csi-driver-webdav/pkg/webdav/mount" + + "k8s.io/klog/v2" +) + +const ( + DefaultDriverName = "webdav.csi.io" + fstype = "davfs" + webdavSharePath = "share" + mountPermissionsField = "mountpermissions" + pvcNameKey = "csi.storage.k8s.io/pvc/name" + pvcNamespaceKey = "csi.storage.k8s.io/pvc/namespace" + pvNameKey = "csi.storage.k8s.io/pv/name" + secretUsernameKey = "username" + secretPasswordKey = "password" +) + +type Driver struct { + name string + nodeID string + endpoint string + version string + mountPermissions uint64 + workingMountDir string + defaultOnDeletePolicy string + + cscap []*csi.ControllerServiceCapability + nscap []*csi.NodeServiceCapability +} + +type DriverOpt struct { + Name string + NodeID string + Endpoint string + MountPermissions uint64 + WorkingMountDir string + DefaultOnDeletePolicy string +} + +func NewDriver(opt *DriverOpt) *Driver { + klog.V(2).Infof("Driver: %v version: %v", opt.Name, driverVersion) + + driverName := opt.Name + if driverName == "" { + driverName = DefaultDriverName + } + + driver := &Driver{ + name: driverName, + nodeID: opt.NodeID, + endpoint: opt.Endpoint, + mountPermissions: opt.MountPermissions, + workingMountDir: opt.WorkingMountDir, + defaultOnDeletePolicy: opt.DefaultOnDeletePolicy, + version: driverName, + } + + driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, + }) + + driver.AddNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{ + csi.NodeServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, + csi.NodeServiceCapability_RPC_UNKNOWN, + }) + + return driver +} + +func (d *Driver) Run() { + versionMeta, err := GetVersionYAML(d.name) + if err != nil { + klog.Fatalf("%v", err) + } + klog.V(2).Infof("\nDRIVER INFORMATION:\n-------------------\n%s\n\nStreaming logs below:", versionMeta) + + mounter := mount.New("") + if runtime.GOOS == "linux" { + // MounterForceUnmounter is only implemented on Linux now + mounter = mounter.(mount.MounterForceUnmounter) + } + server := NewNonBlockingGRPCServer() + server.Start(d.endpoint, + NewIdentityServer(d), + NewControllerServer(d, mounter), + NewNodeServer(d, mounter), + ) + server.Wait() +} + +func (d *Driver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) { + var csc []*csi.ControllerServiceCapability + for _, c := range cl { + csc = append(csc, NewControllerServiceCapability(c)) + } + d.cscap = csc +} + +func (d *Driver) AddNodeServiceCapabilities(nl []csi.NodeServiceCapability_RPC_Type) { + var nsc []*csi.NodeServiceCapability + for _, n := range nl { + nsc = append(nsc, NewNodeServiceCapability(n)) + } + d.nscap = nsc +} diff --git a/pkg/webdav/identity.go b/pkg/webdav/identity.go new file mode 100644 index 0000000..3fa6244 --- /dev/null +++ b/pkg/webdav/identity.go @@ -0,0 +1,69 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webdav + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/protobuf/ptypes/wrappers" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type IdentityServer struct { + Driver *Driver +} + +func NewIdentityServer(d *Driver) *IdentityServer { + return &IdentityServer{ + Driver: d, + } +} + +func (ids *IdentityServer) GetPluginInfo(_ context.Context, _ *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + if ids.Driver.name == "" { + return nil, status.Error(codes.Unavailable, "Driver name not configured") + } + + if ids.Driver.version == "" { + return nil, status.Error(codes.Unavailable, "Driver is missing version") + } + + return &csi.GetPluginInfoResponse{ + Name: ids.Driver.name, + VendorVersion: ids.Driver.version, + }, nil +} + +func (ids *IdentityServer) Probe(_ context.Context, _ *csi.ProbeRequest) (*csi.ProbeResponse, error) { + return &csi.ProbeResponse{Ready: &wrappers.BoolValue{Value: true}}, nil +} + +func (ids *IdentityServer) GetPluginCapabilities(_ context.Context, _ *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + return &csi.GetPluginCapabilitiesResponse{ + Capabilities: []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, + }, + }, + }, + }, + }, nil +} diff --git a/pkg/webdav/mount/mount.go b/pkg/webdav/mount/mount.go new file mode 100644 index 0000000..8917a18 --- /dev/null +++ b/pkg/webdav/mount/mount.go @@ -0,0 +1,405 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// TODO(thockin): This whole pkg is pretty linux-centric. As soon as we have +// an alternate platform, we will need to abstract further. + +package mount + +import ( + "fmt" + "path/filepath" + "strings" + "time" + + utilexec "k8s.io/utils/exec" +) + +const ( + // Default mount command if mounter path is not specified. + defaultMountCommand = "mount" + // Log message where sensitive mount options were removed + sensitiveOptionsRemoved = "" +) + +// Interface defines the set of methods to allow for mount operations on a system. +type Interface interface { + // Mount mounts source to target as fstype with given options. + // options MUST not contain sensitive material (like passwords). + Mount(source string, target string, fstype string, options []string) error + // MountSensitive is the same as Mount() but this method allows + // sensitiveOptions to be passed in a separate parameter from the normal + // mount options and ensures the sensitiveOptions are never logged. This + // method should be used by callers that pass sensitive material (like + // passwords) as mount options. + MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error + // MountSensitiveWithStdin + MountSensitiveWithStdin(source string, target string, fstype string, options []string, sensitiveOptions []string, stdin []string) error + // MountSensitiveWithoutSystemd is the same as MountSensitive() but this method disable using systemd mount. + MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error + // MountSensitiveWithoutSystemdWithMountFlags is the same as MountSensitiveWithoutSystemd() with additional mount flags + MountSensitiveWithoutSystemdWithMountFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error + // Unmount unmounts given target. + Unmount(target string) error + // List returns a list of all mounted filesystems. This can be large. + // On some platforms, reading mounts directly from the OS is not guaranteed + // consistent (i.e. it could change between chunked reads). This is guaranteed + // to be consistent. + List() ([]MountPoint, error) + // IsLikelyNotMountPoint uses heuristics to determine if a directory + // is not a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsLikelyNotMountPoint does NOT properly detect all mountpoint types + // most notably linux bind mounts and symbolic link. For callers that do not + // care about such situations, this is a faster alternative to calling List() + // and scanning that output. + IsLikelyNotMountPoint(file string) (bool, error) + // CanSafelySkipMountPointCheck indicates whether this mounter returns errors on + // operations for targets that are not mount points. If this returns true, no such + // errors will be returned. + CanSafelySkipMountPointCheck() bool + // IsMountPoint determines if a directory is a mountpoint. + // It should return ErrNotExist when the directory does not exist. + // IsMountPoint is more expensive than IsLikelyNotMountPoint. + // IsMountPoint detects bind mounts in linux. + // IsMountPoint may enumerate all the mountpoints using List() and + // the list of mountpoints may be large, then it uses + // isMountPointMatch to evaluate whether the directory is a mountpoint. + IsMountPoint(file string) (bool, error) + // GetMountRefs finds all mount references to pathname, returning a slice of + // paths. Pathname can be a mountpoint path or a normal directory + // (for bind mount). On Linux, pathname is excluded from the slice. + // For example, if /dev/sdc was mounted at /path/a and /path/b, + // GetMountRefs("/path/a") would return ["/path/b"] + // GetMountRefs("/path/b") would return ["/path/a"] + // On Windows there is no way to query all mount points; as long as pathname is + // a valid mount, it will be returned. + GetMountRefs(pathname string) ([]string, error) +} + +// Compile-time check to ensure all Mounter implementations satisfy +// the mount interface. +var _ Interface = &Mounter{} + +type MounterForceUnmounter interface { + Interface + // UnmountWithForce unmounts given target but will retry unmounting with force option + // after given timeout. + UnmountWithForce(target string, umountTimeout time.Duration) error +} + +// MountPoint represents a single line in /proc/mounts or /etc/fstab. +type MountPoint struct { // nolint: golint + Device string + Path string + Type string + Opts []string // Opts may contain sensitive mount options (like passwords) and MUST be treated as such (e.g. not logged). + Freq int + Pass int +} + +type MountErrorType string // nolint: golint + +const ( + FilesystemMismatch MountErrorType = "FilesystemMismatch" + HasFilesystemErrors MountErrorType = "HasFilesystemErrors" + UnformattedReadOnly MountErrorType = "UnformattedReadOnly" + FormatFailed MountErrorType = "FormatFailed" + GetDiskFormatFailed MountErrorType = "GetDiskFormatFailed" + UnknownMountError MountErrorType = "UnknownMountError" +) + +type MountError struct { // nolint: golint + Type MountErrorType + Message string +} + +func (mountError MountError) String() string { + return mountError.Message +} + +func (mountError MountError) Error() string { + return mountError.Message +} + +func NewMountError(mountErrorValue MountErrorType, format string, args ...interface{}) error { + mountError := MountError{ + Type: mountErrorValue, + Message: fmt.Sprintf(format, args...), + } + return mountError +} + +// SafeFormatAndMount probes a device to see if it is formatted. +// Namely it checks to see if a file system is present. If so it +// mounts it otherwise the device is formatted first then mounted. +type SafeFormatAndMount struct { + Interface + Exec utilexec.Interface + + formatSem chan any + formatTimeout time.Duration +} + +func NewSafeFormatAndMount(mounter Interface, exec utilexec.Interface, opts ...Option) *SafeFormatAndMount { + res := &SafeFormatAndMount{ + Interface: mounter, + Exec: exec, + } + for _, opt := range opts { + opt(res) + } + return res +} + +type Option func(*SafeFormatAndMount) + +// WithMaxConcurrentFormat sets the maximum number of concurrent format +// operations executed by the mounter. The timeout controls the maximum +// duration of a format operation before its concurrency token is released. +// Once a token is released, it can be acquired by another concurrent format +// operation. The original operation is allowed to complete. +// If n < 1, concurrency is set to unlimited. +func WithMaxConcurrentFormat(n int, timeout time.Duration) Option { + return func(mounter *SafeFormatAndMount) { + if n > 0 { + mounter.formatSem = make(chan any, n) + mounter.formatTimeout = timeout + } + } +} + +// FormatAndMount formats the given disk, if needed, and mounts it. +// That is if the disk is not formatted and it is not being mounted as +// read-only it will format it first then mount it. Otherwise, if the +// disk is already formatted or it is being mounted as read-only, it +// will be mounted without formatting. +// options MUST not contain sensitive material (like passwords). +func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, fstype string, options []string) error { + return mounter.FormatAndMountSensitive(source, target, fstype, options, nil /* sensitiveOptions */) +} + +// FormatAndMountSensitive is the same as FormatAndMount but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal mount +// options and ensures the sensitiveOptions are never logged. This method should +// be used by callers that pass sensitive material (like passwords) as mount +// options. +func (mounter *SafeFormatAndMount) FormatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return mounter.FormatAndMountSensitiveWithFormatOptions(source, target, fstype, options, sensitiveOptions, nil /* formatOptions */) +} + +// FormatAndMountSensitiveWithFormatOptions behaves exactly the same as +// FormatAndMountSensitive, but allows for options to be passed when the disk +// is formatted. These options are NOT validated in any way and should never +// come directly from untrusted user input as that would be an injection risk. +func (mounter *SafeFormatAndMount) FormatAndMountSensitiveWithFormatOptions(source string, target string, fstype string, options []string, sensitiveOptions []string, formatOptions []string) error { + return mounter.formatAndMountSensitive(source, target, fstype, options, sensitiveOptions, formatOptions) +} + +// getMountRefsByDev finds all references to the device provided +// by mountPath; returns a list of paths. +// Note that mountPath should be path after the evaluation of any symblolic links. +// +//lint:ignore U1000 Ignore unused function temporarily for debugging +func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { + mps, err := mounter.List() + if err != nil { + return nil, err + } + + // Finding the device mounted to mountPath. + diskDev := "" + for i := range mps { + if mountPath == mps[i].Path { + diskDev = mps[i].Device + break + } + } + + // Find all references to the device. + var refs []string + for i := range mps { + if mps[i].Device == diskDev || mps[i].Device == mountPath { + if mps[i].Path != mountPath { + refs = append(refs, mps[i].Path) + } + } + } + return refs, nil +} + +// IsNotMountPoint determines if a directory is a mountpoint. +// It should return ErrNotExist when the directory does not exist. +// IsNotMountPoint is more expensive than IsLikelyNotMountPoint +// and depends on IsMountPoint. +// +// If an error occurs, it returns true (assuming it is not a mountpoint) +// when ErrNotExist is returned for callers similar to IsLikelyNotMountPoint. +// +// Deprecated: This function is kept to keep changes backward compatible with +// previous library version. Callers should prefer mounter.IsMountPoint. +func IsNotMountPoint(mounter Interface, file string) (bool, error) { + isMnt, err := mounter.IsMountPoint(file) + if err != nil { + return true, err + } + return !isMnt, nil +} + +// GetDeviceNameFromMount given a mnt point, find the device from /proc/mounts +// returns the device name, reference count, and error code. +func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) { + mps, err := mounter.List() + if err != nil { + return "", 0, err + } + + // Find the device name. + // FIXME if multiple devices mounted on the same mount path, only the first one is returned. + device := "" + // If mountPath is symlink, need get its target path. + slTarget, err := filepath.EvalSymlinks(mountPath) + if err != nil { + slTarget = mountPath + } + for i := range mps { + if mps[i].Path == slTarget { + device = mps[i].Device + break + } + } + + // Find all references to the device. + refCount := 0 + for i := range mps { + if mps[i].Device == device { + refCount++ + } + } + return device, refCount, nil +} + +// MakeBindOpts detects whether a bind mount is being requested and makes the remount options to +// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. +// The list equals: +// +// options - 'bind' + 'remount' (no duplicate) +func MakeBindOpts(options []string) (bool, []string, []string) { + bind, bindOpts, bindRemountOpts, _ := MakeBindOptsSensitive(options, nil /* sensitiveOptions */) + return bind, bindOpts, bindRemountOpts +} + +// MakeBindOptsSensitive is the same as MakeBindOpts but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal mount +// options and ensures the sensitiveOptions are never logged. This method should +// be used by callers that pass sensitive material (like passwords) as mount +// options. +func MakeBindOptsSensitive(options []string, sensitiveOptions []string) (bool, []string, []string, []string) { + // Because we have an FD opened on the subpath bind mount, the "bind" option + // needs to be included, otherwise the mount target will error as busy if you + // remount as readonly. + // + // As a consequence, all read only bind mounts will no longer change the underlying + // volume mount to be read only. + bindRemountOpts := []string{"bind", "remount"} + bindRemountSensitiveOpts := []string{} + bind := false + bindOpts := []string{"bind"} + + // _netdev is a userspace mount option and does not automatically get added when + // bind mount is created and hence we must carry it over. + if checkForNetDev(options, sensitiveOptions) { + bindOpts = append(bindOpts, "_netdev") + } + + for _, option := range options { + switch option { + case "bind": + bind = true + case "remount": + default: + bindRemountOpts = append(bindRemountOpts, option) + } + } + + for _, sensitiveOption := range sensitiveOptions { + switch sensitiveOption { + case "bind": + bind = true + case "remount": + default: + bindRemountSensitiveOpts = append(bindRemountSensitiveOpts, sensitiveOption) + } + } + + return bind, bindOpts, bindRemountOpts, bindRemountSensitiveOpts +} + +func checkForNetDev(options []string, sensitiveOptions []string) bool { + for _, option := range options { + if option == "_netdev" { + return true + } + } + for _, sensitiveOption := range sensitiveOptions { + if sensitiveOption == "_netdev" { + return true + } + } + return false +} + +// PathWithinBase checks if give path is within given base directory. +func PathWithinBase(fullPath, basePath string) bool { + rel, err := filepath.Rel(basePath, fullPath) + if err != nil { + return false + } + if StartsWithBackstep(rel) { + // Needed to escape the base path. + return false + } + return true +} + +// StartsWithBackstep checks if the given path starts with a backstep segment. +func StartsWithBackstep(rel string) bool { + // normalize to / and check for ../ + return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") +} + +// sanitizedOptionsForLogging will return a comma separated string containing +// options and sensitiveOptions. Each entry in sensitiveOptions will be +// replaced with the string sensitiveOptionsRemoved +// e.g. o1,o2,, +func sanitizedOptionsForLogging(options []string, sensitiveOptions []string) string { + separator := "" + if len(options) > 0 && len(sensitiveOptions) > 0 { + separator = "," + } + + sensitiveOptionsStart := "" + sensitiveOptionsEnd := "" + if len(sensitiveOptions) > 0 { + sensitiveOptionsStart = strings.Repeat(sensitiveOptionsRemoved+",", len(sensitiveOptions)-1) + sensitiveOptionsEnd = sensitiveOptionsRemoved + } + + return strings.Join(options, ",") + + separator + + sensitiveOptionsStart + + sensitiveOptionsEnd +} diff --git a/pkg/webdav/mount/mount_helper_common.go b/pkg/webdav/mount/mount_helper_common.go new file mode 100644 index 0000000..440cf64 --- /dev/null +++ b/pkg/webdav/mount/mount_helper_common.go @@ -0,0 +1,157 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "time" + + "k8s.io/klog/v2" +) + +// CleanupMountPoint unmounts the given path and deletes the remaining directory +// if successful. If extensiveMountPointCheck is true IsNotMountPoint will be +// called instead of IsLikelyNotMountPoint. IsNotMountPoint is more expensive +// but properly handles bind mounts within the same fs. +func CleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool) error { + pathExists, pathErr := PathExists(mountPath) + if !pathExists && pathErr == nil { + klog.Warningf("Warning: mount cleanup skipped because path does not exist: %v", mountPath) + return nil + } + corruptedMnt := IsCorruptedMnt(pathErr) + if pathErr != nil && !corruptedMnt { + return fmt.Errorf("Error checking path: %v", pathErr) + } + return doCleanupMountPoint(mountPath, mounter, extensiveMountPointCheck, corruptedMnt) +} + +func CleanupMountWithForce(mountPath string, mounter MounterForceUnmounter, extensiveMountPointCheck bool, umountTimeout time.Duration) error { + pathExists, pathErr := PathExists(mountPath) + if !pathExists && pathErr == nil { + klog.Warningf("Warning: mount cleanup skipped because path does not exist: %v", mountPath) + return nil + } + corruptedMnt := IsCorruptedMnt(pathErr) + if pathErr != nil && !corruptedMnt { + return fmt.Errorf("Error checking path: %v", pathErr) + } + + if corruptedMnt || mounter.CanSafelySkipMountPointCheck() { + klog.V(4).Infof("unmounting %q (corruptedMount: %t, mounterCanSkipMountPointChecks: %t)", + mountPath, corruptedMnt, mounter.CanSafelySkipMountPointCheck()) + if err := mounter.UnmountWithForce(mountPath, umountTimeout); err != nil { + return err + } + return removePath(mountPath) + } + + notMnt, err := removePathIfNotMountPoint(mountPath, mounter, extensiveMountPointCheck) + // if mountPath is not a mount point, it's just been removed or there was an error + if err != nil || notMnt { + return err + } + + klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath) + if err := mounter.UnmountWithForce(mountPath, umountTimeout); err != nil { + return err + } + + notMnt, err = removePathIfNotMountPoint(mountPath, mounter, extensiveMountPointCheck) + // if mountPath is not a mount point, it's either just been removed or there was an error + if notMnt { + return err + } + // mountPath is still a mount point + return fmt.Errorf("failed to cleanup mount point %v", mountPath) +} + +// doCleanupMountPoint unmounts the given path and +// deletes the remaining directory if successful. +// if extensiveMountPointCheck is true +// IsNotMountPoint will be called instead of IsLikelyNotMountPoint. +// IsNotMountPoint is more expensive but properly handles bind mounts within the same fs. +// if corruptedMnt is true, it means that the mountPath is a corrupted mountpoint, and the mount point check +// will be skipped. The mount point check will also be skipped if the mounter supports it. +func doCleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool, corruptedMnt bool) error { + if corruptedMnt || mounter.CanSafelySkipMountPointCheck() { + klog.V(4).Infof("unmounting %q (corruptedMount: %t, mounterCanSkipMountPointChecks: %t)", + mountPath, corruptedMnt, mounter.CanSafelySkipMountPointCheck()) + if err := mounter.Unmount(mountPath); err != nil { + return err + } + return removePath(mountPath) + } + + notMnt, err := removePathIfNotMountPoint(mountPath, mounter, extensiveMountPointCheck) + // if mountPath is not a mount point, it's just been removed or there was an error + if err != nil || notMnt { + return err + } + + klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath) + if err := mounter.Unmount(mountPath); err != nil { + return err + } + + notMnt, err = removePathIfNotMountPoint(mountPath, mounter, extensiveMountPointCheck) + // if mountPath is not a mount point, it's either just been removed or there was an error + if notMnt { + return err + } + // mountPath is still a mount point + return fmt.Errorf("failed to cleanup mount point %v", mountPath) +} + +// removePathIfNotMountPoint verifies if given mountPath is a mount point if not it attempts +// to remove the directory. Returns true and nil if directory was not a mount point and removed. +func removePathIfNotMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool) (bool, error) { + var notMnt bool + var err error + + if extensiveMountPointCheck { + notMnt, err = IsNotMountPoint(mounter, mountPath) + } else { + notMnt, err = mounter.IsLikelyNotMountPoint(mountPath) + } + + if err != nil { + if os.IsNotExist(err) { + klog.V(4).Infof("%q does not exist", mountPath) + return true, nil + } + return notMnt, err + } + + if notMnt { + klog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath) + return notMnt, os.Remove(mountPath) + } + return notMnt, nil +} + +// removePath attempts to remove the directory. Returns nil if the directory was removed or does not exist. +func removePath(mountPath string) error { + klog.V(4).Infof("Warning: deleting path %q", mountPath) + err := os.Remove(mountPath) + if os.IsNotExist(err) { + klog.V(4).Infof("%q does not exist", mountPath) + return nil + } + return err +} diff --git a/pkg/webdav/mount/mount_helper_unix.go b/pkg/webdav/mount/mount_helper_unix.go new file mode 100644 index 0000000..82210aa --- /dev/null +++ b/pkg/webdav/mount/mount_helper_unix.go @@ -0,0 +1,250 @@ +//go:build !windows +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "sync" + "syscall" + + "golang.org/x/sys/unix" + "k8s.io/klog/v2" + utilio "k8s.io/utils/io" +) + +const ( + // At least number of fields per line in /proc//mountinfo. + expectedAtLeastNumFieldsPerMountInfo = 10 + // How many times to retry for a consistent read of /proc/mounts. + maxListTries = 10 +) + +// IsCorruptedMnt return true if err is about corrupted mount point +func IsCorruptedMnt(err error) bool { + if err == nil { + return false + } + var underlyingError error + switch pe := err.(type) { + case nil: + return false + case *os.PathError: + underlyingError = pe.Err + case *os.LinkError: + underlyingError = pe.Err + case *os.SyscallError: + underlyingError = pe.Err + case syscall.Errno: + underlyingError = err + } + + return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES || underlyingError == syscall.EHOSTDOWN +} + +// MountInfo represents a single line in /proc//mountinfo. +type MountInfo struct { // nolint: golint + // Unique ID for the mount (maybe reused after umount). + ID int + // The ID of the parent mount (or of self for the root of this mount namespace's mount tree). + ParentID int + // Major indicates one half of the device ID which identifies the device class + // (parsed from `st_dev` for files on this filesystem). + Major int + // Minor indicates one half of the device ID which identifies a specific + // instance of device (parsed from `st_dev` for files on this filesystem). + Minor int + // The pathname of the directory in the filesystem which forms the root of this mount. + Root string + // Mount source, filesystem-specific information. e.g. device, tmpfs name. + Source string + // Mount point, the pathname of the mount point. + MountPoint string + // Optional fieds, zero or more fields of the form "tag[:value]". + OptionalFields []string + // The filesystem type in the form "type[.subtype]". + FsType string + // Per-mount options. + MountOptions []string + // Per-superblock options. + SuperOptions []string +} + +// ParseMountInfo parses /proc/xxx/mountinfo. +func ParseMountInfo(filename string) ([]MountInfo, error) { + content, err := readMountInfo(filename) + if err != nil { + return []MountInfo{}, err + } + contentStr := string(content) + infos := []MountInfo{} + + for _, line := range strings.Split(contentStr, "\n") { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + // See `man proc` for authoritative description of format of the file. + fields := strings.Fields(line) + if len(fields) < expectedAtLeastNumFieldsPerMountInfo { + return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line) + } + id, err := strconv.Atoi(fields[0]) + if err != nil { + return nil, err + } + parentID, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err + } + mm := strings.Split(fields[2], ":") + if len(mm) != 2 { + return nil, fmt.Errorf("parsing '%s' failed: unexpected minor:major pair %s", line, mm) + } + major, err := strconv.Atoi(mm[0]) + if err != nil { + return nil, fmt.Errorf("parsing '%s' failed: unable to parse major device id, err:%v", mm[0], err) + } + minor, err := strconv.Atoi(mm[1]) + if err != nil { + return nil, fmt.Errorf("parsing '%s' failed: unable to parse minor device id, err:%v", mm[1], err) + } + + info := MountInfo{ + ID: id, + ParentID: parentID, + Major: major, + Minor: minor, + Root: fields[3], + MountPoint: fields[4], + MountOptions: splitMountOptions(fields[5]), + } + // All fields until "-" are "optional fields". + i := 6 + for ; i < len(fields) && fields[i] != "-"; i++ { + info.OptionalFields = append(info.OptionalFields, fields[i]) + } + // Parse the rest 3 fields. + i++ + if len(fields)-i < 3 { + return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i) + } + info.FsType = fields[i] + info.Source = fields[i+1] + info.SuperOptions = splitMountOptions(fields[i+2]) + infos = append(infos, info) + } + return infos, nil +} + +// splitMountOptions parses comma-separated list of mount options into an array. +// It respects double quotes - commas in them are not considered as the option separator. +func splitMountOptions(s string) []string { + inQuotes := false + list := strings.FieldsFunc(s, func(r rune) bool { + if r == '"' { + inQuotes = !inQuotes + } + // Report a new field only when outside of double quotes. + return r == ',' && !inQuotes + }) + return list +} + +// isMountPointMatch returns true if the path in mp is the same as dir. +// Handles case where mountpoint dir has been renamed due to stale NFS mount. +func isMountPointMatch(mp MountPoint, dir string) bool { + return strings.TrimSuffix(mp.Path, "\\040(deleted)") == dir +} + +// PathExists returns true if the specified path exists. +// TODO: clean this up to use pkg/util/file/FileExists +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } else if errors.Is(err, fs.ErrNotExist) { + err = syscall.Access(path, syscall.F_OK) + if err == nil { + // The access syscall says the file exists, the stat syscall says it + // doesn't. This was observed on CIFS when the path was removed at + // the server somehow. POSIX calls this a stale file handle, let's fake + // that error and treat the path as existing but corrupted. + klog.Warningf("Potential stale file handle detected: %s", path) + return true, syscall.ESTALE + } + return false, nil + } else if IsCorruptedMnt(err) { + return true, err + } + return false, err +} + +// These variables are used solely by kernelHasMountinfoBug. +var ( + hasMountinfoBug bool + checkMountinfoBugOnce sync.Once +) + +// kernelHasMountinfoBug checks if the kernel bug that can lead to incomplete +// mountinfo being read is fixed. It does so by checking the kernel version. +// +// The bug was fixed by the kernel commit 9f6c61f96f2d97 (since Linux 5.8). +// Alas, there is no better way to check if the bug is fixed other than to +// rely on the kernel version returned by uname. +func kernelHasMountinfoBug() bool { + checkMountinfoBugOnce.Do(func() { + // Assume old kernel. + hasMountinfoBug = true + + uname := unix.Utsname{} + err := unix.Uname(&uname) + if err != nil { + return + } + + end := bytes.IndexByte(uname.Release[:], 0) + v := bytes.SplitN(uname.Release[:end], []byte{'.'}, 3) + if len(v) != 3 { + return + } + major, _ := strconv.Atoi(string(v[0])) + minor, _ := strconv.Atoi(string(v[1])) + + if major > 5 || (major == 5 && minor >= 8) { + hasMountinfoBug = false + } + }) + + return hasMountinfoBug +} + +func readMountInfo(path string) ([]byte, error) { + if kernelHasMountinfoBug() { + return utilio.ConsistentRead(path, maxListTries) + } + + return os.ReadFile(path) +} diff --git a/pkg/webdav/mount/mount_linux.go b/pkg/webdav/mount/mount_linux.go new file mode 100644 index 0000000..3281816 --- /dev/null +++ b/pkg/webdav/mount/mount_linux.go @@ -0,0 +1,871 @@ +//go:build linux +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/moby/sys/mountinfo" + + "k8s.io/klog/v2" + utilexec "k8s.io/utils/exec" +) + +const ( + // Number of fields per line in /proc/mounts as per the fstab man page. + expectedNumFieldsPerLine = 6 + // Location of the mount file to use + procMountsPath = "/proc/mounts" + // Location of the mountinfo file + procMountInfoPath = "/proc/self/mountinfo" + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 + // Error thrown by exec cmd.Run() when process spawned by cmd.Start() completes before cmd.Wait() is called (see - k/k issue #103753) + errNoChildProcesses = "wait: no child processes" + // Error returned by some `umount` implementations when the specified path is not a mount point + errNotMounted = "not mounted" +) + +// Mounter provides the default implementation of mount.Interface +// for the linux platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string + withSystemd *bool + trySystemd bool + withSafeNotMountedBehavior bool +} + +var _ MounterForceUnmounter = &Mounter{} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + trySystemd: true, + withSafeNotMountedBehavior: detectSafeNotMountedBehavior(), + } +} + +// NewWithoutSystemd returns a Linux specific mount.Interface for the current +// system. It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. Any +// detection for systemd functionality is disabled with this Mounter. +func NewWithoutSystemd(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + trySystemd: false, + withSafeNotMountedBehavior: detectSafeNotMountedBehavior(), + } +} + +// hasSystemd validates that the withSystemd bool is set, if it is not, +// detectSystemd will be called once for this Mounter instance. +func (mounter *Mounter) hasSystemd() bool { + if !mounter.trySystemd { + mounter.withSystemd = &mounter.trySystemd + } + + if mounter.withSystemd == nil { + withSystemd := detectSystemd() + mounter.withSystemd = &withSystemd + } + + return *mounter.withSystemd +} + +// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must +// be an empty string in case it's not required, e.g. for remount, or for auto filesystem +// type, where kernel handles fstype for you. The mount 'options' is a list of options, +// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is +// required, call Mount with an empty string list or nil. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return mounter.MountSensitive(source, target, fstype, options, nil) +} + +// MountSensitive is the same as Mount() but this method allows +// sensitiveOptions to be passed in a separate parameter from the normal +// mount options and ensures the sensitiveOptions are never logged. This +// method should be used by callers that pass sensitive material (like +// passwords) as mount options. +func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, nil /* mountFlags */, mounter.trySystemd, nil) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, nil /* mountFlags */, mounter.trySystemd, nil) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, nil /* mountFlags */, mounter.trySystemd, nil) +} + +func (mounter *Mounter) MountSensitiveWithStdin(source string, target string, fstype string, options []string, sensitiveOptions []string, stdin []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, nil /* mountFlags */, mounter.trySystemd, stdin) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, nil /* mountFlags */, mounter.trySystemd, stdin) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, nil /* mountFlags */, mounter.trySystemd, stdin) +} + +// MountSensitiveWithoutSystemd is the same as MountSensitive() but disable using systemd mount. +func (mounter *Mounter) MountSensitiveWithoutSystemd(source string, target string, fstype string, options []string, sensitiveOptions []string) error { + return mounter.MountSensitiveWithoutSystemdWithMountFlags(source, target, fstype, options, sensitiveOptions, nil /* mountFlags */) +} + +// MountSensitiveWithoutSystemdWithMountFlags is the same as MountSensitiveWithoutSystemd with additional mount flags. +func (mounter *Mounter) MountSensitiveWithoutSystemdWithMountFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error { + mounterPath := "" + bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts, bindRemountOptsSensitive, mountFlags, false, nil) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts, bindRemountOptsSensitive, mountFlags, false, nil) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := map[string]struct{}{ + "nfs": {}, + "glusterfs": {}, + "ceph": {}, + "cifs": {}, + } + if _, ok := fsTypesNeedMounter[fstype]; ok { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, mountFlags, false, nil) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +// systemdMountRequired is an extension of option to decide whether uses systemd mount. +func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string, systemdMountRequired bool, stdin []string) error { + mountArgs, mountArgsLogStr := MakeMountArgsSensitiveWithMountFlags(source, target, fstype, options, sensitiveOptions, mountFlags) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountArgsLogStr = mountCmd + " " + mountArgsLogStr + mountCmd = mounterPath + } + + if systemdMountRequired && mounter.hasSystemd() { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs, mountArgsLogStr = AddSystemdScopeSensitive("systemd-run", target, mountCmd, mountArgs, mountArgsLogStr) + // } else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + } + + // Logging with sensitive mount options removed. + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgsLogStr) + command := exec.Command(mountCmd, mountArgs...) + if stdin != nil { + writer, err := command.StdinPipe() + if err != nil { + klog.Errorf("Create stdin pipe failed: %v\nMounting command: %s\nMounting arguments: %s\n", err, mountCmd, mountArgsLogStr) + return fmt.Errorf("create stdin pip failed: %v\nMounting command: %s\nMounting arguments: %s", err, mountCmd, mountArgsLogStr) + } + for _, v := range stdin { + io.WriteString(writer, v) + io.WriteString(writer, "\n") + } + writer.Close() + } + + output, err := command.CombinedOutput() + if err != nil { + if err.Error() == errNoChildProcesses { + if command.ProcessState.Success() { + // We don't consider errNoChildProcesses an error if the process itself succeeded (see - k/k issue #103753). + return nil + } + // Rewrite err with the actual exit error of the process. + err = &exec.ExitError{ProcessState: command.ProcessState} + } + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, mountArgsLogStr, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s", + err, mountCmd, mountArgsLogStr, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Infof("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run output: %s, failed with: %v", string(output), err) + return false + } + klog.V(2).Infof("Detected OS with systemd") + return true +} + +// detectSafeNotMountedBehavior returns true if the umount implementation replies "not mounted" +// when the specified path is not mounted. When not sure (permission errors, ...), it returns false. +// When possible, we will trust umount's message and avoid doing our own mount point checks. +// More info: https://github.com/util-linux/util-linux/blob/v2.2/mount/umount.c#L179 +func detectSafeNotMountedBehavior() bool { + return detectSafeNotMountedBehaviorWithExec(utilexec.New()) +} + +// detectSafeNotMountedBehaviorWithExec is for testing with FakeExec. +func detectSafeNotMountedBehaviorWithExec(exec utilexec.Interface) bool { + // create a temp dir and try to umount it + path, err := os.MkdirTemp("", "kubelet-detect-safe-umount") + if err != nil { + klog.V(4).Infof("Cannot create temp dir to detect safe 'not mounted' behavior: %v", err) + return false + } + defer os.RemoveAll(path) + cmd := exec.Command("umount", path) + output, err := cmd.CombinedOutput() + if err != nil { + if strings.Contains(string(output), errNotMounted) { + klog.V(4).Infof("Detected umount with safe 'not mounted' behavior") + return true + } + klog.V(4).Infof("'umount %s' failed with: %v, output: %s", path, err, string(output)) + } + klog.V(4).Infof("Detected umount with unsafe 'not mounted' behavior") + return false +} + +// MakeMountArgs makes the arguments to the mount(8) command. +// options MUST not contain sensitive material (like passwords). +func MakeMountArgs(source, target, fstype string, options []string) (mountArgs []string) { + mountArgs, _ = MakeMountArgsSensitive(source, target, fstype, options, nil /* sensitiveOptions */) + return mountArgs +} + +// MakeMountArgsSensitive makes the arguments to the mount(8) command. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +func MakeMountArgsSensitive(source, target, fstype string, options []string, sensitiveOptions []string) (mountArgs []string, mountArgsLogStr string) { + return MakeMountArgsSensitiveWithMountFlags(source, target, fstype, options, sensitiveOptions, nil /* mountFlags */) +} + +// MakeMountArgsSensitiveWithMountFlags makes the arguments to the mount(8) command. +// sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material) +// mountFlags are additional mount flags that are not related with the fstype +// and mount options +func MakeMountArgsSensitiveWithMountFlags(source, target, fstype string, options []string, sensitiveOptions []string, mountFlags []string) (mountArgs []string, mountArgsLogStr string) { + // Build mount command as follows: + // mount [$mountFlags] [-t $fstype] [-o $options] [$source] $target + mountArgs = []string{} + mountArgsLogStr = "" + + mountArgs = append(mountArgs, mountFlags...) + mountArgsLogStr += strings.Join(mountFlags, " ") + + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + mountArgsLogStr += strings.Join(mountArgs, " ") + } + if len(options) > 0 || len(sensitiveOptions) > 0 { + combinedOptions := []string{} + combinedOptions = append(combinedOptions, options...) + combinedOptions = append(combinedOptions, sensitiveOptions...) + mountArgs = append(mountArgs, "-o", strings.Join(combinedOptions, ",")) + // exclude sensitiveOptions from log string + mountArgsLogStr += " -o " + sanitizedOptionsForLogging(options, sensitiveOptions) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + mountArgsLogStr += " " + source + } + mountArgs = append(mountArgs, target) + mountArgsLogStr += " " + target + + return mountArgs, mountArgsLogStr +} + +// AddSystemdScope adds "system-run --scope" to given command line +// If args contains sensitive material, use AddSystemdScopeSensitive to construct +// a safe to log string. +func AddSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...) +} + +// AddSystemdScopeSensitive adds "system-run --scope" to given command line +// It also accepts takes a sanitized string containing mount arguments, mountArgsLogStr, +// and returns the string appended to the systemd command for logging. +func AddSystemdScopeSensitive(systemdRunPath, mountName, command string, args []string, mountArgsLogStr string) (string, []string, string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...), strings.Join(systemdRunArgs, " ") + " " + mountArgsLogStr +} + +// Unmount unmounts the target. +// If the mounter has safe "not mounted" behavior, no error will be returned when the target is not a mount point. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("Unmounting %s", target) + command := exec.Command("umount", target) + output, err := command.CombinedOutput() + if err != nil { + return checkUmountError(target, command, output, err, mounter.withSafeNotMountedBehavior) + } + return nil +} + +// UnmountWithForce unmounts given target but will retry unmounting with force option +// after given timeout. +func (mounter *Mounter) UnmountWithForce(target string, umountTimeout time.Duration) error { + err := tryUnmount(target, mounter.withSafeNotMountedBehavior, umountTimeout) + if err != nil { + if err == context.DeadlineExceeded { + klog.V(2).Infof("Timed out waiting for unmount of %s, trying with -f", target) + err = forceUmount(target, mounter.withSafeNotMountedBehavior) + } + return err + } + return nil +} + +// List returns a list of all mounted filesystems. +func (*Mounter) List() ([]MountPoint, error) { + return ListProcMounts(procMountsPath) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +// It is fast but not necessarily ALWAYS correct. If the path is in fact +// a bind mount from one part of a mount to another it will not be detected. +// It also can not distinguish between mountpoints and symbolic links. +// mkdir /tmp/a /tmp/b; mount --bind /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") +// will return true. When in fact /tmp/b is a mount point. If this situation +// is of interest to you, don't use this function... +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Stat(file) + if err != nil { + return true, err + } + rootStat, err := os.Stat(filepath.Dir(strings.TrimSuffix(file, "/"))) + if err != nil { + return true, err + } + // If the directory has a different device as parent, then it is a mountpoint. + if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { + return false, nil + } + + return true, nil +} + +// CanSafelySkipMountPointCheck relies on the detected behavior of umount when given a target that is not a mount point. +func (mounter *Mounter) CanSafelySkipMountPointCheck() bool { + return mounter.withSafeNotMountedBehavior +} + +// GetMountRefs finds all mount references to pathname, returns a +// list of paths. Path could be a mountpoint or a normal +// directory (for bind mount). +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + pathExists, pathErr := PathExists(pathname) + if !pathExists { + return []string{}, nil + } else if IsCorruptedMnt(pathErr) { + klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", pathname) + return []string{}, nil + } else if pathErr != nil { + return nil, fmt.Errorf("error checking path %s: %v", pathname, pathErr) + } + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return SearchMountPoints(realpath, procMountInfoPath) +} + +// checkAndRepairFileSystem checks and repairs filesystems using command fsck. +func (mounter *SafeFormatAndMount) checkAndRepairFilesystem(source string) error { + klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) + args := []string{"-a", source} + out, err := mounter.Exec.Command("fsck", args...).CombinedOutput() + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + klog.Infof("Device %s has errors which were corrected by fsck.", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return NewMountError(HasFilesystemErrors, "'fsck' found errors on device %s but could not correct them: %s", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + klog.Infof("`fsck` error %s", string(out)) + default: + klog.Warningf("fsck on device %s failed with error %v, output: %v", source, err, string(out)) + } + } + return nil +} + +// formatAndMount uses unix utils to format and mount the given disk +func (mounter *SafeFormatAndMount) formatAndMountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string, formatOptions []string) error { + readOnly := false + for _, option := range options { + if option == "ro" { + readOnly = true + break + } + } + if !readOnly { + // Check sensitiveOptions for ro + for _, option := range sensitiveOptions { + if option == "ro" { + readOnly = true + break + } + } + } + + options = append(options, "defaults") + mountErrorValue := UnknownMountError + + // Check if the disk is already formatted + existingFormat, err := mounter.GetDiskFormat(source) + if err != nil { + return NewMountError(GetDiskFormatFailed, "failed to get disk format of disk %s: %v", source, err) + } + + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if existingFormat == "" { + // Do not attempt to format the disk if mounting as readonly, return an error to reflect this. + if readOnly { + return NewMountError(UnformattedReadOnly, "cannot mount unformatted disk %s as we are manipulating it in read-only mode", source) + } + + // Disk is unformatted so format it. + args := []string{source} + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + } else if fstype == "xfs" { + args = []string{ + "-f", // force flag + source, + } + } + args = append(formatOptions, args...) + + klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + + output, err := mounter.format(fstype, args) + if err != nil { + // Do not log sensitiveOptions only options + sensitiveOptionsLog := sanitizedOptionsForLogging(options, sensitiveOptions) + detailedErr := fmt.Sprintf("format of disk %q failed: type:(%q) target:(%q) options:(%q) errcode:(%v) output:(%v) ", source, fstype, target, sensitiveOptionsLog, err, string(output)) + klog.Error(detailedErr) + return NewMountError(FormatFailed, detailedErr) + } + + klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) + } else { + if fstype != existingFormat { + // Verify that the disk is formatted with filesystem type we are expecting + mountErrorValue = FilesystemMismatch + klog.Warningf("Configured to mount disk %s as %s but current format is %s, things might break", source, existingFormat, fstype) + } + + if !readOnly { + // Run check tools on the disk to fix repairable issues, only do this for formatted volumes requested as rw. + err := mounter.checkAndRepairFilesystem(source) + if err != nil { + return err + } + } + } + + // Mount the disk + klog.V(4).Infof("Attempting to mount disk %s in %s format at %s", source, fstype, target) + if err := mounter.MountSensitive(source, target, fstype, options, sensitiveOptions); err != nil { + return NewMountError(mountErrorValue, err.Error()) + } + + return nil +} + +func (mounter *SafeFormatAndMount) format(fstype string, args []string) ([]byte, error) { + if mounter.formatSem != nil { + done := make(chan struct{}) + defer close(done) + + mounter.formatSem <- struct{}{} + + go func() { + defer func() { <-mounter.formatSem }() + + timeout := time.NewTimer(mounter.formatTimeout) + defer timeout.Stop() + + select { + case <-done: + case <-timeout.C: + } + }() + } + + return mounter.Exec.Command("mkfs."+fstype, args...).CombinedOutput() +} + +func getDiskFormat(exec utilexec.Interface, disk string) (string, error) { + args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} + klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) + dataOut, err := exec.Command("blkid", args...).CombinedOutput() + output := string(dataOut) + klog.V(4).Infof("Output: %q", output) + + if err != nil { + if exit, ok := err.(utilexec.ExitError); ok { + if exit.ExitStatus() == 2 { + // Disk device is unformatted. + // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was + // not found, or no (specified) devices could be identified, an + // exit code of 2 is returned. + return "", nil + } + } + klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + return "", err + } + + var fstype, pttype string + + lines := strings.Split(output, "\n") + for _, l := range lines { + if len(l) <= 0 { + // Ignore empty line. + continue + } + cs := strings.Split(l, "=") + if len(cs) != 2 { + return "", fmt.Errorf("blkid returns invalid output: %s", output) + } + // TYPE is filesystem type, and PTTYPE is partition table type, according + // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. + if cs[0] == "TYPE" { + fstype = cs[1] + } else if cs[0] == "PTTYPE" { + pttype = cs[1] + } + } + + if len(pttype) > 0 { + klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) + // Returns a special non-empty string as filesystem type, then kubelet + // will not format it. + return "unknown data, probably partitions", nil + } + + return fstype, nil +} + +// GetDiskFormat uses 'blkid' to see if the given disk is unformatted +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { + return getDiskFormat(mounter.Exec, disk) +} + +// ListProcMounts is shared with NsEnterMounter +func ListProcMounts(mountFilePath string) ([]MountPoint, error) { + content, err := readMountInfo(mountFilePath) + if err != nil { + return nil, err + } + return parseProcMounts(content) +} + +func parseProcMounts(content []byte) ([]MountPoint, error) { + out := []MountPoint{} + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + fields := strings.Fields(line) + if len(fields) != expectedNumFieldsPerLine { + // Do not log line in case it contains sensitive Mount options + return nil, fmt.Errorf("wrong number of fields (expected %d, got %d)", expectedNumFieldsPerLine, len(fields)) + } + + mp := MountPoint{ + Device: fields[0], + Path: fields[1], + Type: fields[2], + Opts: strings.Split(fields[3], ","), + } + + freq, err := strconv.Atoi(fields[4]) + if err != nil { + return nil, err + } + mp.Freq = freq + + pass, err := strconv.Atoi(fields[5]) + if err != nil { + return nil, err + } + mp.Pass = pass + + out = append(out, mp) + } + return out, nil +} + +// SearchMountPoints finds all mount references to the source, returns a list of +// mountpoints. +// The source can be a mount point or a normal directory (bind mount). We +// didn't support device because there is no use case by now. +// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, +// it's possible to mount a non-root path of a filesystem, so we need to use +// root path and major:minor to represent mount source uniquely. +// This implementation is shared between Linux and NsEnterMounter +func SearchMountPoints(hostSource, mountInfoPath string) ([]string, error) { + mis, err := ParseMountInfo(mountInfoPath) + if err != nil { + return nil, err + } + + mountID := 0 + rootPath := "" + major := -1 + minor := -1 + + // Finding the underlying root path and major:minor if possible. + // We need search in backward order because it's possible for later mounts + // to overlap earlier mounts. + for i := len(mis) - 1; i >= 0; i-- { + if hostSource == mis[i].MountPoint || PathWithinBase(hostSource, mis[i].MountPoint) { + // If it's a mount point or path under a mount point. + mountID = mis[i].ID + rootPath = filepath.Join(mis[i].Root, strings.TrimPrefix(hostSource, mis[i].MountPoint)) + major = mis[i].Major + minor = mis[i].Minor + break + } + } + + if rootPath == "" || major == -1 || minor == -1 { + return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) + } + + var refs []string + for i := range mis { + if mis[i].ID == mountID { + // Ignore mount entry for mount source itself. + continue + } + if mis[i].Root == rootPath && mis[i].Major == major && mis[i].Minor == minor { + refs = append(refs, mis[i].MountPoint) + } + } + + return refs, nil +} + +// IsMountPoint determines if a file is a mountpoint. +// It first detects bind & any other mountpoints using +// MountedFast function. If the MountedFast function returns +// sure as true and err as nil, then a mountpoint is detected +// successfully. When an error is returned by MountedFast, the +// following is true: +// 1. All errors are returned with IsMountPoint as false +// except os.IsPermission. +// 2. When os.IsPermission is returned by MountedFast, List() +// is called to confirm if the given file is a mountpoint are not. +// +// os.ErrNotExist should always be returned if a file does not exist +// as callers have in past relied on this error and not fallback. +// +// When MountedFast returns sure as false and err as nil (eg: in +// case of bindmounts on kernel version 5.10- ); mounter.List() +// endpoint is called to enumerate all the mountpoints and check if +// it is mountpoint match or not. +func (mounter *Mounter) IsMountPoint(file string) (bool, error) { + isMnt, sure, isMntErr := mountinfo.MountedFast(file) + if sure && isMntErr == nil { + return isMnt, nil + } + if isMntErr != nil { + if errors.Is(isMntErr, fs.ErrNotExist) { + return false, fs.ErrNotExist + } + // We were not allowed to do the simple stat() check, e.g. on NFS with + // root_squash. Fall back to /proc/mounts check below when + // fs.ErrPermission is returned. + if !errors.Is(isMntErr, fs.ErrPermission) { + return false, isMntErr + } + } + // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts. + resolvedFile, err := filepath.EvalSymlinks(file) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, fs.ErrNotExist + } + return false, err + } + + // check all mountpoints since MountedFast is not sure. + // is not reliable for some mountpoint types. + mountPoints, mountPointsErr := mounter.List() + if mountPointsErr != nil { + return false, mountPointsErr + } + for _, mp := range mountPoints { + if isMountPointMatch(mp, resolvedFile) { + return true, nil + } + } + return false, nil +} + +// tryUnmount calls plain "umount" and waits for unmountTimeout for it to finish. +func tryUnmount(target string, withSafeNotMountedBehavior bool, unmountTimeout time.Duration) error { + klog.V(4).Infof("Unmounting %s", target) + ctx, cancel := context.WithTimeout(context.Background(), unmountTimeout) + defer cancel() + + command := exec.CommandContext(ctx, "umount", target) + output, err := command.CombinedOutput() + + // CombinedOutput() does not return DeadlineExceeded, make sure it's + // propagated on timeout. + if ctx.Err() != nil { + return ctx.Err() + } + + if err != nil { + return checkUmountError(target, command, output, err, withSafeNotMountedBehavior) + } + return nil +} + +func forceUmount(target string, withSafeNotMountedBehavior bool) error { + command := exec.Command("umount", "-f", target) + output, err := command.CombinedOutput() + if err != nil { + return checkUmountError(target, command, output, err, withSafeNotMountedBehavior) + } + return nil +} + +// checkUmountError checks a result of umount command and determine a return value. +func checkUmountError(target string, command *exec.Cmd, output []byte, err error, withSafeNotMountedBehavior bool) error { + if err.Error() == errNoChildProcesses { + if command.ProcessState.Success() { + // We don't consider errNoChildProcesses an error if the process itself succeeded (see - k/k issue #103753). + return nil + } + // Rewrite err with the actual exit error of the process. + err = &exec.ExitError{ProcessState: command.ProcessState} + } + if withSafeNotMountedBehavior && strings.Contains(string(output), errNotMounted) { + klog.V(4).Infof("ignoring 'not mounted' error for %s", target) + return nil + } + return fmt.Errorf("unmount failed: %v\nUnmounting arguments: %s\nOutput: %s", err, target, string(output)) +} diff --git a/pkg/webdav/node.go b/pkg/webdav/node.go new file mode 100644 index 0000000..31b48da --- /dev/null +++ b/pkg/webdav/node.go @@ -0,0 +1,155 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webdav + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/klog/v2" + + "github.com/sys-liqian/csi-driver-webdav/pkg/webdav/mount" +) + +type NodeServer struct { + Driver *Driver + mounter mount.Interface +} + +func NewNodeServer(d *Driver, mounter mount.Interface) *NodeServer { + return &NodeServer{ + Driver: d, + mounter: mounter, + } +} + +// NodePublishVolume implements csi.NodeServer. +func (n *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + volCap := req.GetVolumeCapability() + if volCap == nil { + return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request") + } + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + targetPath := req.GetTargetPath() + if len(targetPath) == 0 { + return nil, status.Error(codes.InvalidArgument, "Target path not provided") + } + + mountOptions := volCap.GetMount().GetMountFlags() + if req.GetReadonly() { + mountOptions = append(mountOptions, "ro") + } + + address, subDir, err := ParseVolumeId(volumeID) + if err != nil { + // An invalid ID should be treated as doesn't exist + klog.Warningf("failed to parse volume for volume id %v deletion: %v", volumeID, err) + return &csi.NodePublishVolumeResponse{}, nil + } + + notMnt, err := n.mounter.IsLikelyNotMountPoint(targetPath) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(targetPath, os.FileMode(n.Driver.mountPermissions)); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + notMnt = true + } else { + return nil, status.Error(codes.Internal, err.Error()) + } + } + if !notMnt { + return &csi.NodePublishVolumeResponse{}, nil + } + + sourcePath := filepath.Join(address, subDir) + stdin := []string{req.GetSecrets()[secretUsernameKey], req.GetSecrets()[secretPasswordKey]} + klog.V(2).Infof("NodePublishVolume: volumeID(%v) source(%s) targetPath(%s) mountflags(%v)", volumeID, sourcePath, targetPath, mountOptions) + err = n.mounter.MountSensitiveWithStdin(sourcePath, targetPath, fstype, mountOptions, nil, stdin) + if err != nil { + if os.IsPermission(err) { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + if strings.Contains(err.Error(), "invalid argument") { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.NodePublishVolumeResponse{}, nil +} + +// NodeUnpublishVolume implements csi.NodeServer. +func (n *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + targetPath := req.GetTargetPath() + if len(targetPath) == 0 { + return nil, status.Error(codes.InvalidArgument, "Target path missing in request") + } + klog.V(2).Infof("NodeUnpublishVolume: unmounting volume %s on %s", volumeID, targetPath) + + err := umount(n.mounter, targetPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to unmount target %q: %v", targetPath, err) + } + + klog.V(2).Infof("NodeUnpublishVolume: unmount volume %s on %s successfully", volumeID, targetPath) + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +// NodeGetInfo implements csi.NodeServer. +func (n *NodeServer) NodeGetInfo(context.Context, *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + return &csi.NodeGetInfoResponse{NodeId: n.Driver.nodeID}, nil +} + +// NodeGetCapabilities implements csi.NodeServer. +func (n *NodeServer) NodeGetCapabilities(context.Context, *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + return &csi.NodeGetCapabilitiesResponse{ + Capabilities: n.Driver.nscap, + }, nil +} + +// NodeExpandVolume implements csi.NodeServer. +func (*NodeServer) NodeExpandVolume(context.Context, *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// NodeGetVolumeStats implements csi.NodeServer. +func (*NodeServer) NodeGetVolumeStats(context.Context, *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// NodeStageVolume implements csi.NodeServer. +func (*NodeServer) NodeStageVolume(context.Context, *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// NodeUnstageVolume implements csi.NodeServer. +func (*NodeServer) NodeUnstageVolume(context.Context, *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} diff --git a/pkg/webdav/server.go b/pkg/webdav/server.go new file mode 100644 index 0000000..7ced1ad --- /dev/null +++ b/pkg/webdav/server.go @@ -0,0 +1,108 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webdav + +import ( + "net" + "os" + "sync" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc" + "k8s.io/klog/v2" +) + +// Defines Non blocking GRPC server interfaces +type NonBlockingGRPCServer interface { + // Start services at the endpoint + Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) + // Waits for the service to stop + Wait() + // Stops the service gracefully + Stop() + // Stops the service forcefully + ForceStop() +} + +func NewNonBlockingGRPCServer() NonBlockingGRPCServer { + return &nonBlockingGRPCServer{} +} + +// NonBlocking server +type nonBlockingGRPCServer struct { + wg sync.WaitGroup + server *grpc.Server +} + +func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { + s.wg.Add(1) + go s.serve(endpoint, ids, cs, ns) +} + +func (s *nonBlockingGRPCServer) Wait() { + s.wg.Wait() +} + +func (s *nonBlockingGRPCServer) Stop() { + s.server.GracefulStop() +} + +func (s *nonBlockingGRPCServer) ForceStop() { + s.server.Stop() +} + +func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { + proto, addr, err := ParseEndpoint(endpoint) + if err != nil { + klog.Fatal(err.Error()) + } + + if proto == "unix" { + addr = "/" + addr + if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { + klog.Fatalf("Failed to remove %s, error: %s", addr, err.Error()) + } + } + + listener, err := net.Listen(proto, addr) + if err != nil { + klog.Fatalf("Failed to listen: %v", err) + } + + opts := []grpc.ServerOption{ + grpc.UnaryInterceptor(logGRPC), + } + server := grpc.NewServer(opts...) + s.server = server + + if ids != nil { + csi.RegisterIdentityServer(server, ids) + } + if cs != nil { + csi.RegisterControllerServer(server, cs) + } + if ns != nil { + csi.RegisterNodeServer(server, ns) + } + + klog.Infof("Listening for connections on address: %#v", listener.Addr()) + + err = server.Serve(listener) + if err != nil { + klog.Fatalf("Failed to serve grpc server: %v", err) + } +} diff --git a/pkg/webdav/utils.go b/pkg/webdav/utils.go new file mode 100644 index 0000000..c3bbcdb --- /dev/null +++ b/pkg/webdav/utils.go @@ -0,0 +1,107 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webdav + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/sys-liqian/csi-driver-webdav/pkg/webdav/mount" + "google.golang.org/grpc" + "k8s.io/klog/v2" +) + +func ParseEndpoint(ep string) (string, string, error) { + if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") { + s := strings.SplitN(ep, "://", 2) + if s[1] != "" { + return s[0], s[1], nil + } + } + return "", "", fmt.Errorf("invalid endpoint: %v", ep) +} + +func getLogLevel(method string) int32 { + if method == "/csi.v1.Identity/Probe" || + method == "/csi.v1.Node/NodeGetCapabilities" || + method == "/csi.v1.Node/NodeGetVolumeStats" { + return 8 + } + return 2 +} + +func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + level := klog.Level(getLogLevel(info.FullMethod)) + klog.V(level).Infof("GRPC call: %s", info.FullMethod) + klog.V(level).Infof("GRPC request: %s", req) + + resp, err := handler(ctx, req) + if err != nil { + klog.Errorf("GRPC error: %v", err) + } else { + klog.V(level).Infof("GRPC response: %s", resp) + } + return resp, err +} + +func NewControllerServiceCapability(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability { + return &csi.ControllerServiceCapability{ + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: cap, + }, + }, + } +} + +func NewNodeServiceCapability(cap csi.NodeServiceCapability_RPC_Type) *csi.NodeServiceCapability { + return &csi.NodeServiceCapability{ + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: cap, + }, + }, + } +} + +func MakeVolumeId(webdavSharePath, volumeName string) string { + return fmt.Sprintf("%s#%s", webdavSharePath, volumeName) +} + +func ParseVolumeId(volumeId string) (webdavSharePath, subDir string, err error) { + arr := strings.Split(volumeId, "#") + if len(arr) < 2 { + return "", "", errors.New("invalid volumeId") + } + return arr[0], arr[1], nil +} + +func umount(mounter mount.Interface, path string) (err error) { + extensiveMountPointCheck := true + forceUnmounter, ok := mounter.(mount.MounterForceUnmounter) + if ok { + klog.V(2).Infof("force unmount %s", path) + err = mount.CleanupMountWithForce(path, forceUnmounter, extensiveMountPointCheck, 30*time.Second) + } else { + err = mount.CleanupMountPoint(path, mounter, extensiveMountPointCheck) + } + return +} diff --git a/pkg/webdav/version.go b/pkg/webdav/version.go new file mode 100644 index 0000000..ea735d2 --- /dev/null +++ b/pkg/webdav/version.go @@ -0,0 +1,66 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webdav + +import ( + "fmt" + "runtime" + "strings" + + "sigs.k8s.io/yaml" +) + +// These are set during build time via -ldflags +var ( + driverVersion = "N/A" + gitCommit = "N/A" + buildDate = "N/A" +) + +// VersionInfo holds the version information of the driver +type VersionInfo struct { + DriverName string `json:"Driver Name"` + DriverVersion string `json:"Driver Version"` + GitCommit string `json:"Git Commit"` + BuildDate string `json:"Build Date"` + GoVersion string `json:"Go Version"` + Compiler string `json:"Compiler"` + Platform string `json:"Platform"` +} + +// GetVersion returns the version information of the driver +func GetVersion(driverName string) VersionInfo { + return VersionInfo{ + DriverName: driverName, + DriverVersion: driverVersion, + GitCommit: gitCommit, + BuildDate: buildDate, + GoVersion: runtime.Version(), + Compiler: runtime.Compiler, + Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), + } +} + +// GetVersionYAML returns the version information of the driver +func GetVersionYAML(driverName string) (string, error) { + info := GetVersion(driverName) + marshalled, err := yaml.Marshal(&info) + if err != nil { + return "", err + } + return strings.TrimSpace(string(marshalled)), nil +} diff --git a/test/kind.sh b/test/kind.sh new file mode 100644 index 0000000..00887ed --- /dev/null +++ b/test/kind.sh @@ -0,0 +1 @@ +kind create cluster --config=kind.yaml \ No newline at end of file diff --git a/test/kind.yaml b/test/kind.yaml new file mode 100644 index 0000000..fb4b936 --- /dev/null +++ b/test/kind.yaml @@ -0,0 +1,18 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + image: kindest/node:v1.29.0 + extraMounts: + - hostPath: /root/workspace/csi-driver-webdav/test/csi + containerPath: /csi +networking: + apiServerPort: 6443 + podSubnet: 172.16.0.0/16 + serviceSubnet: 172.19.0.0/16 +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://hub-mirror.c.163.com"] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"] + endpoint = ["http://registry:5000"] \ No newline at end of file