I0327 01:59:07.052106 1 main.go:96] Version: v4.2.0 I0327 01:59:07.053625 1 connection.go:154] Connecting to unix:///csi/csi-provisioner.sock I0327 01:59:09.152550 1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginInfo I0327 01:59:09.152796 1 connection.go:184] GRPC request: {} I0327 01:59:09.160654 1 connection.go:186] GRPC response: {"name":"rbd.csi.ceph.com","vendor_version":"v3.5.1"} I0327 01:59:09.160816 1 connection.go:187] GRPC error: I0327 01:59:09.160888 1 main.go:154] CSI driver name: "rbd.csi.ceph.com" I0327 01:59:09.160978 1 common.go:111] Probing CSI driver for readiness I0327 01:59:09.161029 1 connection.go:183] GRPC call: /csi.v1.Identity/Probe I0327 01:59:09.161059 1 connection.go:184] GRPC request: {} I0327 01:59:09.161728 1 connection.go:186] GRPC response: {} I0327 01:59:09.161822 1 connection.go:187] GRPC error: I0327 01:59:09.161831 1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerGetCapabilities I0327 01:59:09.161836 1 connection.go:184] GRPC request: {} I0327 01:59:09.162698 1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":5}}},{"Type":{"Rpc":{"type":7}}},{"Type":{"Rpc":{"type":9}}}]} I0327 01:59:09.162831 1 connection.go:187] GRPC error: I0327 01:59:09.162843 1 main.go:192] Start NewCSISnapshotSideCarController with snapshotter [rbd.csi.ceph.com] kubeconfig [] csiTimeout [1m0s] csiAddress [unix:///csi/csi-provisioner.sock] resyncPeriod [15m0s] snapshotNamePrefix [snapshot] snapshotNameUUIDLength [824634935016] I0327 01:59:09.164222 1 leaderelection.go:248] attempting to acquire leader lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com... I0327 01:59:09.173979 1 leaderelection.go:258] successfully acquired lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:09.174191 1 leader_election.go:212] new leader detected, current leader: ceph-csi-rbd-provisioner-6f6497f79c-xmq9z I0327 01:59:09.174655 1 leader_election.go:205] became leader, starting I0327 01:59:09.174970 1 reflector.go:219] Starting reflector *v1.VolumeSnapshotClass (15m0s) from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0327 01:59:09.174984 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0327 01:59:09.175151 1 reflector.go:219] Starting reflector *v1.VolumeSnapshotContent (15m0s) from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0327 01:59:09.175181 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0327 01:59:09.175239 1 snapshot_controller_base.go:115] Starting CSI snapshotter E0327 01:59:09.176319 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) E0327 01:59:09.176514 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 01:59:09.181511 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:10.511227 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:10.513084 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 01:59:10.729831 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:10.731736 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 01:59:12.814499 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:12.816229 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 01:59:13.011848 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:13.013614 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 01:59:14.190988 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:16.423775 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:16.425689 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 01:59:18.214729 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:18.216537 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 01:59:19.231644 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:23.827855 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:23.829325 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 01:59:24.242368 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:25.237510 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:25.239189 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 01:59:29.252865 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:34.260529 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:39.269872 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:40.481532 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:40.482827 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 01:59:44.277932 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:44.634814 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 01:59:44.636709 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 01:59:49.286248 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:54.295171 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 01:59:59.304344 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:04.313513 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:09.322397 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:14.333263 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:15.722508 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:00:15.724421 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 02:00:19.341758 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:24.350352 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:26.912804 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:00:26.914562 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 02:00:29.359064 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:34.369043 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:39.376731 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:44.386248 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:49.397370 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:54.406766 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:00:57.145011 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:00:57.147013 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 02:00:59.416754 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:04.428511 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:06.457139 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:01:06.458865 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 02:01:09.437506 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:14.447150 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:19.456348 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:24.465582 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:29.473301 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:34.480855 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:39.491499 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:41.214037 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:01:41.215749 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 02:01:44.499836 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:44.949959 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:01:44.952415 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 02:01:49.511859 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:54.521168 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:01:59.528355 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:04.543297 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:09.553006 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:14.561765 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:19.574248 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:20.009438 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:02:20.011487 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 02:02:24.583630 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:29.592445 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:34.610317 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:35.327139 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:02:35.328500 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0327 02:02:39.618668 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:44.626835 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:49.633504 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:54.638738 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:02:56.570219 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0327 02:02:56.572531 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0327 02:02:59.647775 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0327 02:03:04.655841 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com