I0511 07:33:10.634994 49646 cephcsi.go:191] Driver version: v3.11.0 and Git version: bc24b5eca87626d690a29effa9d7420cc0154a7a I0511 07:33:10.635181 49646 cephcsi.go:268] Initial PID limit is set to -1 I0511 07:33:10.635214 49646 cephcsi.go:274] Reconfigured PID limit to -1 (max) I0511 07:33:10.635235 49646 cephcsi.go:223] Starting driver type: rbd with name: rbd.csi.ceph.com I0511 07:33:10.647930 49646 mount_linux.go:282] Detected umount with safe 'not mounted' behavior I0511 07:33:10.675666 49646 rbd_attach.go:242] nbd module loaded I0511 07:33:10.675730 49646 rbd_attach.go:256] kernel version "5.15.0-177-generic" supports cookie feature I0511 07:33:10.699691 49646 rbd_attach.go:272] rbd-nbd tool supports cookie feature I0511 07:33:10.700317 49646 server.go:114] listening for CSI-Addons requests on address: &net.UnixAddr{Name:"/csi/csi-addons.sock", Net:"unix"} I0511 07:33:10.700522 49646 server.go:117] Listening for connections on address: &net.UnixAddr{Name:"//csi/csi.sock", Net:"unix"} I0511 07:33:11.456913 49646 utils.go:198] ID: 1 GRPC call: /csi.v1.Identity/GetPluginInfo I0511 07:33:11.459227 49646 utils.go:199] ID: 1 GRPC request: {} I0511 07:33:11.459260 49646 identityserver-default.go:40] ID: 1 Using default GetPluginInfo I0511 07:33:11.459343 49646 utils.go:205] ID: 1 GRPC response: {"name":"rbd.csi.ceph.com","vendor_version":"v3.11.0"} I0511 07:33:11.957840 49646 utils.go:198] ID: 2 GRPC call: /csi.v1.Node/NodeGetInfo I0511 07:33:11.957982 49646 utils.go:199] ID: 2 GRPC request: {} I0511 07:33:11.958001 49646 nodeserver-default.go:45] ID: 2 Using default NodeGetInfo I0511 07:33:11.958133 49646 utils.go:205] ID: 2 GRPC response: {"accessible_topology":{},"node_id":"instance"} I0511 07:33:25.867874 49646 utils.go:198] ID: 3 GRPC call: /csi.v1.Node/NodeGetCapabilities I0511 07:33:25.868042 49646 utils.go:199] ID: 3 GRPC request: {} I0511 07:33:25.868290 49646 utils.go:205] ID: 3 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0511 07:33:25.874895 49646 utils.go:198] ID: 4 GRPC call: /csi.v1.Node/NodeGetCapabilities I0511 07:33:25.874971 49646 utils.go:199] ID: 4 GRPC request: {} I0511 07:33:25.875152 49646 utils.go:205] ID: 4 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0511 07:33:25.876156 49646 utils.go:198] ID: 5 GRPC call: /csi.v1.Node/NodeGetCapabilities I0511 07:33:25.876205 49646 utils.go:199] ID: 5 GRPC request: {} I0511 07:33:25.876371 49646 utils.go:205] ID: 5 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0511 07:33:25.880310 49646 utils.go:198] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC call: /csi.v1.Node/NodeStageVolume I0511 07:33:25.880897 49646 utils.go:199] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC request: {"secrets":"***stripped***","staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["discard"]}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"4837cbf8-4f90-4300-b3f6-726c9b9f89b4","imageFeatures":"layering","imageName":"csi-vol-313f67e5-5440-4d72-889b-ade53397224e","journalPool":"kube","pool":"kube","storage.kubernetes.io/csiProvisionerIdentity":"1778484792706-9273-rbd.csi.ceph.com"},"volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e"} I0511 07:33:25.900209 49646 omap.go:89] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e got omap values: (pool="kube", namespace="", name="csi.volume.313f67e5-5440-4d72-889b-ade53397224e"): map[csi.imageid:37b4fcedec31 csi.imagename:csi-vol-313f67e5-5440-4d72-889b-ade53397224e csi.volname:pvc-14039edc-8089-45cd-8812-ae1f30c58238 csi.volume.owner:default] I0511 07:33:25.946387 49646 rbd_util.go:352] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e checking for ImageFeatures: [layering] I0511 07:33:25.977233 49646 cephcmds.go:105] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e command succeeded: rbd [device list --format=json --device-type krbd] I0511 07:33:26.000155 49646 rbd_attach.go:437] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e rbd: map mon 199.204.45.242 I0511 07:33:26.085202 49646 cephcmds.go:105] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e command succeeded: rbd [--id kube -m 199.204.45.242 --keyfile=***stripped*** map kube/csi-vol-313f67e5-5440-4d72-889b-ade53397224e --device-type krbd --options noudev] I0511 07:33:26.085335 49646 nodeserver.go:422] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e rbd image: kube/csi-vol-313f67e5-5440-4d72-889b-ade53397224e was successfully mapped at /dev/rbd0 I0511 07:33:26.085510 49646 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0511 07:33:26.096127 49646 mount_linux.go:580] Output: "" I0511 07:33:26.179888 49646 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0511 07:33:26.192217 49646 mount_linux.go:580] Output: "DEVNAME=/dev/rbd0\nTYPE=ext4\n" I0511 07:33:26.192334 49646 mount_linux.go:439] Checking for issues with fsck on disk: /dev/rbd0 I0511 07:33:26.224082 49646 mount_linux.go:544] Attempting to mount disk /dev/rbd0 in ext4 format at /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e I0511 07:33:26.224205 49646 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o _netdev,discard,defaults /dev/rbd0 /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e) I0511 07:33:26.242038 49646 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0511 07:33:26.249273 49646 mount_linux.go:580] Output: "DEVNAME=/dev/rbd0\nTYPE=ext4\n" I0511 07:33:26.249339 49646 resizefs_linux.go:137] ResizeFs.needResize - checking mounted volume /dev/rbd0 I0511 07:33:26.252015 49646 resizefs_linux.go:141] Ext size: filesystem size=5368709120, block size=4096 I0511 07:33:26.252039 49646 resizefs_linux.go:156] Volume /dev/rbd0: device size=5368709120, filesystem size=5368709120, block size=4096 I0511 07:33:26.252062 49646 nodeserver.go:382] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e rbd: successfully mounted volume 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e to stagingTargetPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e I0511 07:33:26.252323 49646 utils.go:205] ID: 6 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC response: {} I0511 07:33:26.253869 49646 utils.go:198] ID: 7 GRPC call: /csi.v1.Node/NodeGetCapabilities I0511 07:33:26.253948 49646 utils.go:199] ID: 7 GRPC request: {} I0511 07:33:26.254050 49646 utils.go:205] ID: 7 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0511 07:33:26.257655 49646 utils.go:198] ID: 8 GRPC call: /csi.v1.Node/NodeGetCapabilities I0511 07:33:26.257755 49646 utils.go:199] ID: 8 GRPC request: {} I0511 07:33:26.257846 49646 utils.go:205] ID: 8 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0511 07:33:26.259015 49646 utils.go:198] ID: 9 GRPC call: /csi.v1.Node/NodeGetCapabilities I0511 07:33:26.259058 49646 utils.go:199] ID: 9 GRPC request: {} I0511 07:33:26.259148 49646 utils.go:205] ID: 9 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0511 07:33:26.260109 49646 utils.go:198] ID: 10 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC call: /csi.v1.Node/NodePublishVolume I0511 07:33:26.260313 49646 utils.go:199] ID: 10 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount","target_path":"/var/lib/kubelet/pods/b6f4f218-3536-4a6e-b6b3-4ed2dcbde2f7/volumes/kubernetes.io~csi/pvc-14039edc-8089-45cd-8812-ae1f30c58238/mount","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["discard"]}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"4837cbf8-4f90-4300-b3f6-726c9b9f89b4","imageFeatures":"layering","imageName":"csi-vol-313f67e5-5440-4d72-889b-ade53397224e","journalPool":"kube","pool":"kube","storage.kubernetes.io/csiProvisionerIdentity":"1778484792706-9273-rbd.csi.ceph.com"},"volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e"} I0511 07:33:26.260467 49646 nodeserver.go:859] ID: 10 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e target /var/lib/kubelet/pods/b6f4f218-3536-4a6e-b6b3-4ed2dcbde2f7/volumes/kubernetes.io~csi/pvc-14039edc-8089-45cd-8812-ae1f30c58238/mount isBlock false fstype ext4 stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e readonly false mountflags [bind _netdev discard] I0511 07:33:26.260517 49646 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o bind,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e /var/lib/kubelet/pods/b6f4f218-3536-4a6e-b6b3-4ed2dcbde2f7/volumes/kubernetes.io~csi/pvc-14039edc-8089-45cd-8812-ae1f30c58238/mount) I0511 07:33:26.266282 49646 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o bind,remount,_netdev,discard /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e /var/lib/kubelet/pods/b6f4f218-3536-4a6e-b6b3-4ed2dcbde2f7/volumes/kubernetes.io~csi/pvc-14039edc-8089-45cd-8812-ae1f30c58238/mount) I0511 07:33:26.268595 49646 nodeserver.go:742] ID: 10 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e rbd: successfully mounted stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e to targetPath /var/lib/kubelet/pods/b6f4f218-3536-4a6e-b6b3-4ed2dcbde2f7/volumes/kubernetes.io~csi/pvc-14039edc-8089-45cd-8812-ae1f30c58238/mount I0511 07:33:26.268681 49646 utils.go:205] ID: 10 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC response: {} I0511 07:33:36.649643 49646 utils.go:198] ID: 11 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC call: /csi.v1.Node/NodeUnpublishVolume I0511 07:33:36.649847 49646 utils.go:199] ID: 11 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC request: {"target_path":"/var/lib/kubelet/pods/b6f4f218-3536-4a6e-b6b3-4ed2dcbde2f7/volumes/kubernetes.io~csi/pvc-14039edc-8089-45cd-8812-ae1f30c58238/mount","volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e"} I0511 07:33:36.650006 49646 mount_linux.go:360] Unmounting /var/lib/kubelet/pods/b6f4f218-3536-4a6e-b6b3-4ed2dcbde2f7/volumes/kubernetes.io~csi/pvc-14039edc-8089-45cd-8812-ae1f30c58238/mount I0511 07:33:36.655814 49646 nodeserver.go:945] ID: 11 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e rbd: successfully unbound volume 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e from /var/lib/kubelet/pods/b6f4f218-3536-4a6e-b6b3-4ed2dcbde2f7/volumes/kubernetes.io~csi/pvc-14039edc-8089-45cd-8812-ae1f30c58238/mount I0511 07:33:36.655920 49646 utils.go:205] ID: 11 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC response: {} I0511 07:33:36.752556 49646 utils.go:198] ID: 12 GRPC call: /csi.v1.Node/NodeGetCapabilities I0511 07:33:36.752606 49646 utils.go:199] ID: 12 GRPC request: {} I0511 07:33:36.752730 49646 utils.go:205] ID: 12 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0511 07:33:36.754461 49646 utils.go:198] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC call: /csi.v1.Node/NodeUnstageVolume I0511 07:33:36.754870 49646 utils.go:199] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount","volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e"} I0511 07:33:36.755087 49646 mount_linux.go:360] Unmounting /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e I0511 07:33:37.443508 49646 nodeserver.go:1001] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e successfully unmounted volume (0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e) from staging path (/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4d584882ead97ce402672da48db1f43ef52b5e48cb74e4ac531abc09dbfe67db/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e) I0511 07:33:37.506330 49646 cephcmds.go:105] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e command succeeded: rbd [unmap kube/csi-vol-313f67e5-5440-4d72-889b-ade53397224e --device-type krbd --options noudev] I0511 07:33:37.506381 49646 nodeserver.go:1059] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e successfully unmapped volume (0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e) I0511 07:33:37.506634 49646 utils.go:205] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-313f67e5-5440-4d72-889b-ade53397224e GRPC response: {}