I0804 15:26:47.890306 1 cephcsi.go:191] Driver version: v3.11.0 and Git version: bc24b5eca87626d690a29effa9d7420cc0154a7a I0804 15:26:47.890420 1 cephcsi.go:223] Starting driver type: controller with name: rbd.csi.ceph.com I0804 15:26:47.890909 1 leaderelection.go:250] attempting to acquire leader lease kube-system/rbd.csi.ceph.com-kube-system... I0804 15:26:47.903324 1 leaderelection.go:260] successfully acquired lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:26:47.907070 1 reflector.go:289] Starting reflector *v1.PersistentVolume (9h50m23.837808029s) from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0804 15:26:47.907102 1 reflector.go:325] Listing and watching *v1.PersistentVolume from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0804 15:26:47.909927 1 reflector.go:351] Caches populated for *v1.PersistentVolume from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0804 15:26:47.910826 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:26:49.919660 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:26:52.403477 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:26:54.410692 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:26:56.418904 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:26:58.429424 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:00.439352 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:02.448727 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:04.454909 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:06.463419 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:08.471964 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:10.485722 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:12.497743 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:14.504678 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:16.513023 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:18.521754 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:20.529693 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:22.542414 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:24.549793 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:26.556675 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:28.566805 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:30.577774 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:32.586806 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:34.593723 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:36.612204 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:38.618604 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:40.630393 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:42.637162 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:44.653001 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:46.661644 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:48.670754 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:50.678485 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:52.688601 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:54.697029 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:56.705614 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:27:58.715874 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:00.724049 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:02.750719 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:04.760997 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:06.770169 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:08.783743 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:10.791583 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:12.800156 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:14.807948 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:16.815689 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:18.823986 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:20.835232 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:22.843476 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:24.854331 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:26.864456 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:28.873902 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:30.881655 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:32.892132 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:34.903836 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:36.911386 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:38.917425 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:40.925084 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:42.934330 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:44.943107 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:46.952265 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:48.958154 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:50.964861 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:52.973420 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:54.980151 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:56.990413 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:28:58.998034 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:01.012286 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:03.020049 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:05.028569 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:07.039932 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:09.052214 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:11.061812 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:13.068258 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:15.078200 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:17.086763 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:19.096043 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:21.106475 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:23.115088 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:25.124829 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:27.133706 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:29.141824 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:31.156255 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:33.162798 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:35.171919 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:37.181501 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:39.191203 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:41.200413 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:43.208748 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:45.217624 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:47.227551 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:49.321645 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:51.377240 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:53.385840 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:55.394146 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:57.412113 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:29:59.421468 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:01.428882 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:03.438108 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:05.446192 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:07.451914 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:09.483068 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:11.491397 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:13.498559 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:15.505679 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:17.515475 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:19.524053 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:21.535742 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:23.542104 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:25.550865 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:27.575751 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:29.584421 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:31.593961 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:33.600888 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:35.608289 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:37.620562 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:39.629277 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:41.640276 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:43.651606 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:45.672042 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:47.682768 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:49.693333 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:51.705144 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:53.713453 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system [controller-runtime] log.SetLogger(...) was never called; logs will not be displayed. Detected at: > goroutine 103 [running]: > runtime/debug.Stack() > /usr/local/go/src/runtime/debug/stack.go:24 +0x5e > sigs.k8s.io/controller-runtime/pkg/log.eventuallyFulfillRoot() > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/log/log.go:60 +0xcd > sigs.k8s.io/controller-runtime/pkg/log.(*delegatingLogSink).WithValues(0xc0009863c0, {0xc0009669c0, 0x6, 0x6}) > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/log/deleg.go:168 +0x49 > github.com/go-logr/logr.Logger.WithValues(...) > /go/src/github.com/ceph/ceph-csi/vendor/github.com/go-logr/logr/logr.go:332 > sigs.k8s.io/controller-runtime/pkg/controller.NewUnmanaged.func1(0xc000590020) > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/controller/controller.go:121 +0x1d7 > sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc0000fd220, {0x27c0488, 0xc0004e8aa0}, {0x1f93660?, 0xc000619fe0?}) > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:306 +0x16a > sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc0000fd220, {0x27c0488, 0xc0004e8aa0}) > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:266 +0x1af > sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2() > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:227 +0x79 > created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2 in goroutine 42 > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:223 +0x565 I0804 15:30:53.823890 1 reflector.go:289] Starting reflector *v1.Secret (10h0m3.0910656s) from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0804 15:30:53.823902 1 reflector.go:325] Listing and watching *v1.Secret from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0804 15:30:53.863353 1 reflector.go:351] Caches populated for *v1.Secret from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0804 15:30:53.945524 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-a41386c8-39da-43d3-997c-a25abcc0ec90:8d10f924-7254-4e08-a97e-023bef201fdd] I0804 15:30:53.947649 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.8d10f924-7254-4e08-a97e-023bef201fdd"): map[csi.imageid:37f388078efb csi.imagename:csi-vol-8d10f924-7254-4e08-a97e-023bef201fdd csi.volname:pvc-a41386c8-39da-43d3-997c-a25abcc0ec90 csi.volume.owner:default] I0804 15:30:54.001284 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-a41386c8-39da-43d3-997c-a25abcc0ec90:8d10f924-7254-4e08-a97e-023bef201fdd] I0804 15:30:54.002177 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.8d10f924-7254-4e08-a97e-023bef201fdd"): map[csi.imageid:37f388078efb csi.imagename:csi-vol-8d10f924-7254-4e08-a97e-023bef201fdd csi.volname:pvc-a41386c8-39da-43d3-997c-a25abcc0ec90 csi.volume.owner:default] I0804 15:30:55.721353 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:57.735022 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:30:59.743289 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:01.752134 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:03.763112 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:05.770468 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:07.779247 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:09.785924 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:11.794634 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:13.993640 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:16.002575 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:18.008570 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0804 15:31:20.021006 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system E0804 15:31:32.021457 1 leaderelection.go:332] error retrieving resource lock kube-system/rbd.csi.ceph.com-kube-system: Get "https://10.96.0.1:443/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/rbd.csi.ceph.com-kube-system": context deadline exceeded I0804 15:31:32.021498 1 leaderelection.go:285] failed to renew lease kube-system/rbd.csi.ceph.com-kube-system: timed out waiting for the condition E0804 15:31:32.021548 1 controller.go:85] failed to start manager leader election lost E0804 15:31:32.021555 1 cephcsi.go:314] leader election lost