I0331 18:31:51.967294 1 cephcsi.go:191] Driver version: v3.11.0 and Git version: bc24b5eca87626d690a29effa9d7420cc0154a7a I0331 18:31:51.967425 1 cephcsi.go:223] Starting driver type: controller with name: rbd.csi.ceph.com I0331 18:31:51.968139 1 leaderelection.go:250] attempting to acquire leader lease kube-system/rbd.csi.ceph.com-kube-system... E0331 18:32:21.969018 1 leaderelection.go:332] error retrieving resource lock kube-system/rbd.csi.ceph.com-kube-system: Get "https://10.96.0.1:443/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/rbd.csi.ceph.com-kube-system": dial tcp 10.96.0.1:443: i/o timeout I0331 18:32:21.969087 1 leaderelection.go:255] failed to acquire lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:26.249964 1 leaderelection.go:260] successfully acquired lease kube-system/rbd.csi.ceph.com-kube-system [controller-runtime] log.SetLogger(...) was never called; logs will not be displayed. Detected at: > goroutine 61 [running]: > runtime/debug.Stack() > /usr/local/go/src/runtime/debug/stack.go:24 +0x5e > sigs.k8s.io/controller-runtime/pkg/log.eventuallyFulfillRoot() > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/log/log.go:60 +0xcd > sigs.k8s.io/controller-runtime/pkg/log.(*delegatingLogSink).Enabled(0xc00053c340, 0x10?) > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/log/deleg.go:111 +0x32 > github.com/go-logr/logr.Logger.Info({{0x27c4810?, 0xc00053c340?}, 0xc0007087a8?}, {0xc0000bae40, 0x5b}, {0xc000591500, 0x6, 0x6}) > /go/src/github.com/ceph/ceph-csi/vendor/github.com/go-logr/logr/logr.go:276 +0x72 > sigs.k8s.io/controller-runtime/pkg/internal/recorder.(*Provider).getBroadcaster.func1.1(0xc000796a00) > /go/src/github.com/ceph/ceph-csi/vendor/sigs.k8s.io/controller-runtime/pkg/internal/recorder/recorder.go:104 +0x1e7 > k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1() > /go/src/github.com/ceph/ceph-csi/vendor/k8s.io/client-go/tools/record/event.go:401 +0x11d > created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher in goroutine 69 > /go/src/github.com/ceph/ceph-csi/vendor/k8s.io/client-go/tools/record/event.go:387 +0x109 I0331 18:32:26.255353 1 reflector.go:289] Starting reflector *v1.PersistentVolume (10h18m57.930338386s) from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0331 18:32:26.255399 1 reflector.go:325] Listing and watching *v1.PersistentVolume from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0331 18:32:26.256561 1 reflector.go:351] Caches populated for *v1.PersistentVolume from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0331 18:32:26.367595 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:28.400098 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:30.503833 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:32.566157 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:34.604754 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:36.613786 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:38.622373 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:40.631717 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:42.641958 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:44.766386 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:46.777162 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:48.786189 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:50.795291 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:52.804834 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:54.814932 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:56.826439 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:32:58.866698 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:00.877629 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:02.888649 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:04.900603 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:06.911049 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:08.930540 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:10.938366 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:13.525369 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:15.534497 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:17.546305 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:19.610297 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:21.618983 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:23.877064 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:25.885585 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:27.899632 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:29.909238 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:31.919747 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:33.928385 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:35.935828 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:37.945504 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:39.952574 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:41.966784 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:43.977680 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:46.017428 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:48.116067 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:50.127343 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:52.137234 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:54.145977 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:56.153294 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:33:58.326973 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:00.632936 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:02.642161 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:04.649845 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:06.667596 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:08.676746 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:10.684886 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:12.691929 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:14.701328 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:16.709860 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:18.718191 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:20.726547 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:22.735863 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:24.744670 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:26.754252 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:28.764900 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:30.774644 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:32.783535 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:34.791798 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:36.800675 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:38.808926 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:40.815913 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:42.828768 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:44.838467 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:46.849117 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:48.859702 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:50.868546 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:53.090941 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:55.358559 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:57.405031 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:34:59.563362 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:00.830381 1 reflector.go:289] Starting reflector *v1.Secret (10h16m1.402649728s) from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0331 18:35:00.830419 1 reflector.go:325] Listing and watching *v1.Secret from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0331 18:35:00.874724 1 reflector.go:351] Caches populated for *v1.Secret from sigs.k8s.io/controller-runtime/pkg/cache/internal/informers.go:105 I0331 18:35:00.950559 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-314c66d5-52e7-4d97-9899-34f09ebce8ec:814c4017-a01a-44b7-af0e-0b32fa668d00] I0331 18:35:00.952026 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.814c4017-a01a-44b7-af0e-0b32fa668d00"): map[csi.imageid:37d6de4af617 csi.imagename:csi-vol-814c4017-a01a-44b7-af0e-0b32fa668d00 csi.volname:pvc-314c66d5-52e7-4d97-9899-34f09ebce8ec csi.volume.owner:default] I0331 18:35:01.045212 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volumes.default"): map[csi.volume.pvc-314c66d5-52e7-4d97-9899-34f09ebce8ec:814c4017-a01a-44b7-af0e-0b32fa668d00] I0331 18:35:01.046339 1 omap.go:89] got omap values: (pool="kube", namespace="", name="csi.volume.814c4017-a01a-44b7-af0e-0b32fa668d00"): map[csi.imageid:37d6de4af617 csi.imagename:csi-vol-814c4017-a01a-44b7-af0e-0b32fa668d00 csi.volname:pvc-314c66d5-52e7-4d97-9899-34f09ebce8ec csi.volume.owner:default] I0331 18:35:01.574397 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:03.583290 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:05.593867 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:07.606861 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:09.639910 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:11.672852 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:13.792710 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:15.938192 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:18.907136 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:21.043536 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:23.092393 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:25.358973 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:27.371105 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:29.382248 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:31.389902 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:33.610847 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:35.622551 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:37.640413 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:39.649746 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:41.663922 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:43.671053 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:45.687056 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:47.694962 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:49.701263 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:51.707427 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:53.716204 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:55.724686 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:57.733682 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:35:59.744133 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:36:01.924106 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system I0331 18:36:03.936677 1 leaderelection.go:281] successfully renewed lease kube-system/rbd.csi.ceph.com-kube-system