I0428 06:57:11.582853 1 main.go:96] Version: v4.2.0 I0428 06:57:11.585135 1 connection.go:154] Connecting to unix:///csi/csi-provisioner.sock I0428 06:57:13.635433 1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginInfo I0428 06:57:13.635453 1 connection.go:184] GRPC request: {} I0428 06:57:13.640957 1 connection.go:186] GRPC response: {"name":"rbd.csi.ceph.com","vendor_version":"v3.5.1"} I0428 06:57:13.641147 1 connection.go:187] GRPC error: I0428 06:57:13.641165 1 main.go:154] CSI driver name: "rbd.csi.ceph.com" I0428 06:57:13.641176 1 common.go:111] Probing CSI driver for readiness I0428 06:57:13.641189 1 connection.go:183] GRPC call: /csi.v1.Identity/Probe I0428 06:57:13.641196 1 connection.go:184] GRPC request: {} I0428 06:57:13.642963 1 connection.go:186] GRPC response: {} I0428 06:57:13.643111 1 connection.go:187] GRPC error: I0428 06:57:13.643128 1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerGetCapabilities I0428 06:57:13.643134 1 connection.go:184] GRPC request: {} I0428 06:57:13.644465 1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":5}}},{"Type":{"Rpc":{"type":7}}},{"Type":{"Rpc":{"type":9}}}]} I0428 06:57:13.644632 1 connection.go:187] GRPC error: I0428 06:57:13.644646 1 main.go:192] Start NewCSISnapshotSideCarController with snapshotter [rbd.csi.ceph.com] kubeconfig [] csiTimeout [1m0s] csiAddress [unix:///csi/csi-provisioner.sock] resyncPeriod [15m0s] snapshotNamePrefix [snapshot] snapshotNameUUIDLength [824640897112] I0428 06:57:13.645955 1 leaderelection.go:248] attempting to acquire leader lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com... I0428 06:57:13.660285 1 leaderelection.go:258] successfully acquired lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:13.660378 1 leader_election.go:212] new leader detected, current leader: ceph-csi-rbd-provisioner-6f6497f79c-jzj9v I0428 06:57:13.660424 1 leader_election.go:205] became leader, starting I0428 06:57:13.661075 1 reflector.go:219] Starting reflector *v1.VolumeSnapshotContent (15m0s) from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0428 06:57:13.661160 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0428 06:57:13.661313 1 snapshot_controller_base.go:115] Starting CSI snapshotter I0428 06:57:13.661443 1 reflector.go:219] Starting reflector *v1.VolumeSnapshotClass (15m0s) from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0428 06:57:13.661474 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:13.665015 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) E0428 06:57:13.665312 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:57:13.669971 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:14.998104 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:15.000606 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:57:15.218145 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:15.220285 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 06:57:17.301212 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:17.302786 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:57:17.500281 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:17.502326 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 06:57:18.861475 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:20.913018 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:20.914884 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 06:57:22.700991 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:22.702989 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:57:23.868998 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:28.317611 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:28.319368 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 06:57:28.875763 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:29.724093 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:29.725168 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:57:33.893964 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:38.905885 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:43.913723 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:44.971677 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:44.973598 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 06:57:48.932690 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:49.120945 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:57:49.123397 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:57:53.942393 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:57:58.952248 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:03.960744 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:08.969625 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:13.979613 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:18.996461 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:20.209553 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:58:20.211868 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:58:24.004710 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:29.012589 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:31.403867 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:58:31.405816 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 06:58:34.020615 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:39.030627 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:44.040173 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:49.048489 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:54.058540 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:58:59.068834 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:01.631805 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:59:01.634155 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:59:04.079805 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:09.087203 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:10.948251 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:59:10.950357 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 06:59:14.098400 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:19.109156 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:24.116993 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:29.127720 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:34.135141 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:39.146136 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:44.156568 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:45.701595 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:59:45.703713 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 06:59:49.165717 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:49.441698 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 06:59:49.443687 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 06:59:54.348726 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 06:59:59.358158 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:04.367700 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:09.377822 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:14.387780 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:19.401253 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:24.415177 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:24.497768 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 07:00:24.500654 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 07:00:29.427856 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:34.638623 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:39.649660 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:39.817152 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 07:00:39.819264 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 07:00:44.668259 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:49.676064 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:54.683519 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:00:59.692888 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:01:01.059464 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 07:01:01.061185 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 07:01:04.700238 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:01:09.708676 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:01:14.913747 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 07:01:15.916107 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 07:01:15.917785 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 07:01:19.927964 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com