I0413 14:33:07.762056 42154 cephcsi.go:191] Driver version: v3.11.0 and Git version: bc24b5eca87626d690a29effa9d7420cc0154a7a I0413 14:33:07.762306 42154 cephcsi.go:268] Initial PID limit is set to -1 I0413 14:33:07.762346 42154 cephcsi.go:274] Reconfigured PID limit to -1 (max) I0413 14:33:07.762373 42154 cephcsi.go:223] Starting driver type: rbd with name: rbd.csi.ceph.com I0413 14:33:07.774442 42154 mount_linux.go:282] Detected umount with safe 'not mounted' behavior I0413 14:33:07.805235 42154 rbd_attach.go:242] nbd module loaded I0413 14:33:07.805294 42154 rbd_attach.go:256] kernel version "5.15.0-174-generic" supports cookie feature I0413 14:33:07.831724 42154 rbd_attach.go:272] rbd-nbd tool supports cookie feature I0413 14:33:07.832809 42154 server.go:114] listening for CSI-Addons requests on address: &net.UnixAddr{Name:"/csi/csi-addons.sock", Net:"unix"} I0413 14:33:07.832974 42154 server.go:117] Listening for connections on address: &net.UnixAddr{Name:"//csi/csi.sock", Net:"unix"} I0413 14:33:08.583387 42154 utils.go:198] ID: 1 GRPC call: /csi.v1.Identity/GetPluginInfo I0413 14:33:08.585207 42154 utils.go:199] ID: 1 GRPC request: {} I0413 14:33:08.585236 42154 identityserver-default.go:40] ID: 1 Using default GetPluginInfo I0413 14:33:08.585292 42154 utils.go:205] ID: 1 GRPC response: {"name":"rbd.csi.ceph.com","vendor_version":"v3.11.0"} I0413 14:33:08.647653 42154 utils.go:198] ID: 2 GRPC call: /csi.v1.Node/NodeGetInfo I0413 14:33:08.647746 42154 utils.go:199] ID: 2 GRPC request: {} I0413 14:33:08.647760 42154 nodeserver-default.go:45] ID: 2 Using default NodeGetInfo I0413 14:33:08.647835 42154 utils.go:205] ID: 2 GRPC response: {"accessible_topology":{},"node_id":"instance"} I0413 14:34:08.741417 42154 utils.go:198] ID: 3 GRPC call: /csi.v1.Identity/Probe I0413 14:34:08.741572 42154 utils.go:199] ID: 3 GRPC request: {} I0413 14:34:08.741753 42154 utils.go:205] ID: 3 GRPC response: {} I0413 14:35:08.747836 42154 utils.go:198] ID: 4 GRPC call: /csi.v1.Identity/Probe I0413 14:35:08.748139 42154 utils.go:199] ID: 4 GRPC request: {} I0413 14:35:08.748182 42154 utils.go:205] ID: 4 GRPC response: {} I0413 14:36:08.787732 42154 utils.go:198] ID: 5 GRPC call: /csi.v1.Identity/Probe I0413 14:36:08.787798 42154 utils.go:199] ID: 5 GRPC request: {} I0413 14:36:08.787820 42154 utils.go:205] ID: 5 GRPC response: {} I0413 14:36:31.928523 42154 utils.go:198] ID: 6 GRPC call: /csi.v1.Node/NodeGetCapabilities I0413 14:36:31.929063 42154 utils.go:199] ID: 6 GRPC request: {} I0413 14:36:31.929372 42154 utils.go:205] ID: 6 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0413 14:36:31.936762 42154 utils.go:198] ID: 7 GRPC call: /csi.v1.Node/NodeGetCapabilities I0413 14:36:31.936805 42154 utils.go:199] ID: 7 GRPC request: {} I0413 14:36:31.936927 42154 utils.go:205] ID: 7 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0413 14:36:31.938344 42154 utils.go:198] ID: 8 GRPC call: /csi.v1.Node/NodeGetCapabilities I0413 14:36:31.938387 42154 utils.go:199] ID: 8 GRPC request: {} I0413 14:36:31.938560 42154 utils.go:205] ID: 8 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0413 14:36:31.940116 42154 utils.go:198] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC call: /csi.v1.Node/NodeStageVolume I0413 14:36:31.940285 42154 utils.go:199] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC request: {"secrets":"***stripped***","staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["discard"]}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"4837cbf8-4f90-4300-b3f6-726c9b9f89b4","imageFeatures":"layering","imageName":"csi-vol-58727333-c783-454f-93b7-e56316d40acc","journalPool":"kube","pool":"kube","storage.kubernetes.io/csiProvisionerIdentity":"1776090791111-2275-rbd.csi.ceph.com"},"volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc"} I0413 14:36:31.956233 42154 omap.go:89] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc got omap values: (pool="kube", namespace="", name="csi.volume.58727333-c783-454f-93b7-e56316d40acc"): map[csi.imageid:37d939abe71a csi.imagename:csi-vol-58727333-c783-454f-93b7-e56316d40acc csi.volname:pvc-e114e412-34c0-4ca0-a585-130270f464d2 csi.volume.owner:default] I0413 14:36:32.150562 42154 rbd_util.go:352] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc checking for ImageFeatures: [layering] I0413 14:36:32.183105 42154 cephcmds.go:105] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc command succeeded: rbd [device list --format=json --device-type krbd] I0413 14:36:32.209006 42154 rbd_attach.go:437] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc rbd: map mon 199.19.213.191 I0413 14:36:32.291231 42154 cephcmds.go:105] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc command succeeded: rbd [--id kube -m 199.19.213.191 --keyfile=***stripped*** map kube/csi-vol-58727333-c783-454f-93b7-e56316d40acc --device-type krbd --options noudev] I0413 14:36:32.291289 42154 nodeserver.go:422] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc rbd image: kube/csi-vol-58727333-c783-454f-93b7-e56316d40acc was successfully mapped at /dev/rbd0 I0413 14:36:32.291399 42154 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0413 14:36:32.303782 42154 mount_linux.go:580] Output: "" I0413 14:36:32.418032 42154 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0413 14:36:32.431404 42154 mount_linux.go:580] Output: "DEVNAME=/dev/rbd0\nTYPE=ext4\n" I0413 14:36:32.431438 42154 mount_linux.go:439] Checking for issues with fsck on disk: /dev/rbd0 I0413 14:36:32.470258 42154 mount_linux.go:544] Attempting to mount disk /dev/rbd0 in ext4 format at /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc I0413 14:36:32.470324 42154 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o _netdev,discard,defaults /dev/rbd0 /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc) I0413 14:36:32.491241 42154 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0413 14:36:32.502640 42154 mount_linux.go:580] Output: "DEVNAME=/dev/rbd0\nTYPE=ext4\n" I0413 14:36:32.502663 42154 resizefs_linux.go:137] ResizeFs.needResize - checking mounted volume /dev/rbd0 I0413 14:36:32.505285 42154 resizefs_linux.go:141] Ext size: filesystem size=5368709120, block size=4096 I0413 14:36:32.505303 42154 resizefs_linux.go:156] Volume /dev/rbd0: device size=5368709120, filesystem size=5368709120, block size=4096 I0413 14:36:32.505323 42154 nodeserver.go:382] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc rbd: successfully mounted volume 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc to stagingTargetPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc I0413 14:36:32.505652 42154 utils.go:205] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC response: {} I0413 14:36:32.507362 42154 utils.go:198] ID: 10 GRPC call: /csi.v1.Node/NodeGetCapabilities I0413 14:36:32.507411 42154 utils.go:199] ID: 10 GRPC request: {} I0413 14:36:32.507611 42154 utils.go:205] ID: 10 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0413 14:36:32.513256 42154 utils.go:198] ID: 11 GRPC call: /csi.v1.Node/NodeGetCapabilities I0413 14:36:32.513310 42154 utils.go:199] ID: 11 GRPC request: {} I0413 14:36:32.513439 42154 utils.go:205] ID: 11 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0413 14:36:32.515448 42154 utils.go:198] ID: 12 GRPC call: /csi.v1.Node/NodeGetCapabilities I0413 14:36:32.515504 42154 utils.go:199] ID: 12 GRPC request: {} I0413 14:36:32.515641 42154 utils.go:205] ID: 12 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0413 14:36:32.517055 42154 utils.go:198] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC call: /csi.v1.Node/NodePublishVolume I0413 14:36:32.517226 42154 utils.go:199] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount","target_path":"/var/lib/kubelet/pods/3480c419-06df-4da8-bd15-255824c94169/volumes/kubernetes.io~csi/pvc-e114e412-34c0-4ca0-a585-130270f464d2/mount","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["discard"]}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"4837cbf8-4f90-4300-b3f6-726c9b9f89b4","imageFeatures":"layering","imageName":"csi-vol-58727333-c783-454f-93b7-e56316d40acc","journalPool":"kube","pool":"kube","storage.kubernetes.io/csiProvisionerIdentity":"1776090791111-2275-rbd.csi.ceph.com"},"volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc"} I0413 14:36:32.517363 42154 nodeserver.go:859] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc target /var/lib/kubelet/pods/3480c419-06df-4da8-bd15-255824c94169/volumes/kubernetes.io~csi/pvc-e114e412-34c0-4ca0-a585-130270f464d2/mount isBlock false fstype ext4 stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc readonly false mountflags [bind _netdev discard] I0413 14:36:32.517387 42154 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o bind,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc /var/lib/kubelet/pods/3480c419-06df-4da8-bd15-255824c94169/volumes/kubernetes.io~csi/pvc-e114e412-34c0-4ca0-a585-130270f464d2/mount) I0413 14:36:32.522573 42154 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o bind,remount,_netdev,discard /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc /var/lib/kubelet/pods/3480c419-06df-4da8-bd15-255824c94169/volumes/kubernetes.io~csi/pvc-e114e412-34c0-4ca0-a585-130270f464d2/mount) I0413 14:36:32.524255 42154 nodeserver.go:742] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc rbd: successfully mounted stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc to targetPath /var/lib/kubelet/pods/3480c419-06df-4da8-bd15-255824c94169/volumes/kubernetes.io~csi/pvc-e114e412-34c0-4ca0-a585-130270f464d2/mount I0413 14:36:32.524328 42154 utils.go:205] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC response: {} I0413 14:36:41.198143 42154 utils.go:198] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC call: /csi.v1.Node/NodeUnpublishVolume I0413 14:36:41.198246 42154 utils.go:199] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC request: {"target_path":"/var/lib/kubelet/pods/3480c419-06df-4da8-bd15-255824c94169/volumes/kubernetes.io~csi/pvc-e114e412-34c0-4ca0-a585-130270f464d2/mount","volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc"} I0413 14:36:41.198344 42154 mount_linux.go:360] Unmounting /var/lib/kubelet/pods/3480c419-06df-4da8-bd15-255824c94169/volumes/kubernetes.io~csi/pvc-e114e412-34c0-4ca0-a585-130270f464d2/mount I0413 14:36:41.202519 42154 nodeserver.go:945] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc rbd: successfully unbound volume 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc from /var/lib/kubelet/pods/3480c419-06df-4da8-bd15-255824c94169/volumes/kubernetes.io~csi/pvc-e114e412-34c0-4ca0-a585-130270f464d2/mount I0413 14:36:41.202567 42154 utils.go:205] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC response: {} I0413 14:36:41.301223 42154 utils.go:198] ID: 15 GRPC call: /csi.v1.Node/NodeGetCapabilities I0413 14:36:41.301326 42154 utils.go:199] ID: 15 GRPC request: {} I0413 14:36:41.301471 42154 utils.go:205] ID: 15 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0413 14:36:41.302892 42154 utils.go:198] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC call: /csi.v1.Node/NodeUnstageVolume I0413 14:36:41.303055 42154 utils.go:199] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount","volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc"} I0413 14:36:41.303169 42154 mount_linux.go:360] Unmounting /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc I0413 14:36:41.362915 42154 nodeserver.go:1001] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc successfully unmounted volume (0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc) from staging path (/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/e2dff35b1106c21c60f551a72532d146ed6a3562c971b0781833f199070123ac/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc) I0413 14:36:41.438344 42154 cephcmds.go:105] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc command succeeded: rbd [unmap kube/csi-vol-58727333-c783-454f-93b7-e56316d40acc --device-type krbd --options noudev] I0413 14:36:41.438393 42154 nodeserver.go:1059] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc successfully unmapped volume (0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc) I0413 14:36:41.438582 42154 utils.go:205] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-58727333-c783-454f-93b7-e56316d40acc GRPC response: {} I0413 14:37:08.755804 42154 utils.go:198] ID: 17 GRPC call: /csi.v1.Identity/Probe I0413 14:37:08.755849 42154 utils.go:199] ID: 17 GRPC request: {} I0413 14:37:08.755865 42154 utils.go:205] ID: 17 GRPC response: {}