I0215 14:16:41.077744 40638 cephcsi.go:191] Driver version: v3.11.0 and Git version: bc24b5eca87626d690a29effa9d7420cc0154a7a I0215 14:16:41.078052 40638 cephcsi.go:268] Initial PID limit is set to -1 I0215 14:16:41.078176 40638 cephcsi.go:274] Reconfigured PID limit to -1 (max) I0215 14:16:41.078228 40638 cephcsi.go:223] Starting driver type: rbd with name: rbd.csi.ceph.com I0215 14:16:41.087212 40638 mount_linux.go:282] Detected umount with safe 'not mounted' behavior I0215 14:16:41.189970 40638 rbd_attach.go:242] nbd module loaded I0215 14:16:41.190007 40638 rbd_attach.go:256] kernel version "5.15.0-170-generic" supports cookie feature I0215 14:16:41.219349 40638 rbd_attach.go:272] rbd-nbd tool supports cookie feature I0215 14:16:41.220013 40638 server.go:114] listening for CSI-Addons requests on address: &net.UnixAddr{Name:"/csi/csi-addons.sock", Net:"unix"} I0215 14:16:41.222163 40638 server.go:117] Listening for connections on address: &net.UnixAddr{Name:"//csi/csi.sock", Net:"unix"} I0215 14:16:42.257837 40638 utils.go:198] ID: 1 GRPC call: /csi.v1.Identity/GetPluginInfo I0215 14:16:42.258855 40638 utils.go:199] ID: 1 GRPC request: {} I0215 14:16:42.258872 40638 identityserver-default.go:40] ID: 1 Using default GetPluginInfo I0215 14:16:42.261534 40638 utils.go:205] ID: 1 GRPC response: {"name":"rbd.csi.ceph.com","vendor_version":"v3.11.0"} I0215 14:16:43.176611 40638 utils.go:198] ID: 2 GRPC call: /csi.v1.Node/NodeGetInfo I0215 14:16:43.176677 40638 utils.go:199] ID: 2 GRPC request: {} I0215 14:16:43.176687 40638 nodeserver-default.go:45] ID: 2 Using default NodeGetInfo I0215 14:16:43.176734 40638 utils.go:205] ID: 2 GRPC response: {"accessible_topology":{},"node_id":"instance"} I0215 14:17:42.559151 40638 utils.go:198] ID: 3 GRPC call: /csi.v1.Identity/Probe I0215 14:17:42.559678 40638 utils.go:199] ID: 3 GRPC request: {} I0215 14:17:42.559918 40638 utils.go:205] ID: 3 GRPC response: {} I0215 14:18:42.498620 40638 utils.go:198] ID: 4 GRPC call: /csi.v1.Identity/Probe I0215 14:18:42.498657 40638 utils.go:199] ID: 4 GRPC request: {} I0215 14:18:42.498668 40638 utils.go:205] ID: 4 GRPC response: {} I0215 14:19:42.498694 40638 utils.go:198] ID: 5 GRPC call: /csi.v1.Identity/Probe I0215 14:19:42.498748 40638 utils.go:199] ID: 5 GRPC request: {} I0215 14:19:42.498774 40638 utils.go:205] ID: 5 GRPC response: {} I0215 14:19:48.923907 40638 utils.go:198] ID: 6 GRPC call: /csi.v1.Node/NodeGetCapabilities I0215 14:19:48.924190 40638 utils.go:199] ID: 6 GRPC request: {} I0215 14:19:48.924436 40638 utils.go:205] ID: 6 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0215 14:19:48.931173 40638 utils.go:198] ID: 7 GRPC call: /csi.v1.Node/NodeGetCapabilities I0215 14:19:48.931195 40638 utils.go:199] ID: 7 GRPC request: {} I0215 14:19:48.931410 40638 utils.go:205] ID: 7 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0215 14:19:48.932268 40638 utils.go:198] ID: 8 GRPC call: /csi.v1.Node/NodeGetCapabilities I0215 14:19:48.932372 40638 utils.go:199] ID: 8 GRPC request: {} I0215 14:19:48.932555 40638 utils.go:205] ID: 8 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0215 14:19:48.934073 40638 utils.go:198] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC call: /csi.v1.Node/NodeStageVolume I0215 14:19:48.934475 40638 utils.go:199] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC request: {"secrets":"***stripped***","staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["discard"]}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"4837cbf8-4f90-4300-b3f6-726c9b9f89b4","imageFeatures":"layering","imageName":"csi-vol-340fa294-d995-4408-932c-94959a1899ac","journalPool":"kube","pool":"kube","storage.kubernetes.io/csiProvisionerIdentity":"1771165005101-2828-rbd.csi.ceph.com"},"volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac"} I0215 14:19:48.953732 40638 omap.go:89] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac got omap values: (pool="kube", namespace="", name="csi.volume.340fa294-d995-4408-932c-94959a1899ac"): map[csi.imageid:37d97a28bb72 csi.imagename:csi-vol-340fa294-d995-4408-932c-94959a1899ac csi.volname:pvc-c54a276b-989e-44f7-b7d0-38715293d752 csi.volume.owner:default] I0215 14:19:48.998337 40638 rbd_util.go:352] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac checking for ImageFeatures: [layering] I0215 14:19:49.047305 40638 cephcmds.go:105] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac command succeeded: rbd [device list --format=json --device-type krbd] I0215 14:19:49.067142 40638 rbd_attach.go:437] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac rbd: map mon 162.253.55.218 I0215 14:19:49.138933 40638 cephcmds.go:105] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac command succeeded: rbd [--id kube -m 162.253.55.218 --keyfile=***stripped*** map kube/csi-vol-340fa294-d995-4408-932c-94959a1899ac --device-type krbd --options noudev] I0215 14:19:49.139015 40638 nodeserver.go:422] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac rbd image: kube/csi-vol-340fa294-d995-4408-932c-94959a1899ac was successfully mapped at /dev/rbd0 I0215 14:19:49.139146 40638 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0215 14:19:49.148877 40638 mount_linux.go:580] Output: "" I0215 14:19:49.230743 40638 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0215 14:19:49.241616 40638 mount_linux.go:580] Output: "DEVNAME=/dev/rbd0\nTYPE=ext4\n" I0215 14:19:49.241640 40638 mount_linux.go:439] Checking for issues with fsck on disk: /dev/rbd0 I0215 14:19:49.262919 40638 mount_linux.go:544] Attempting to mount disk /dev/rbd0 in ext4 format at /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac I0215 14:19:49.262962 40638 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o _netdev,discard,defaults /dev/rbd0 /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac) I0215 14:19:49.276915 40638 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0215 14:19:49.290085 40638 mount_linux.go:580] Output: "DEVNAME=/dev/rbd0\nTYPE=ext4\n" I0215 14:19:49.290292 40638 resizefs_linux.go:137] ResizeFs.needResize - checking mounted volume /dev/rbd0 I0215 14:19:49.293299 40638 resizefs_linux.go:141] Ext size: filesystem size=5368709120, block size=4096 I0215 14:19:49.293312 40638 resizefs_linux.go:156] Volume /dev/rbd0: device size=5368709120, filesystem size=5368709120, block size=4096 I0215 14:19:49.293345 40638 nodeserver.go:382] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac rbd: successfully mounted volume 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac to stagingTargetPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac I0215 14:19:49.293442 40638 utils.go:205] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC response: {} I0215 14:19:49.294680 40638 utils.go:198] ID: 10 GRPC call: /csi.v1.Node/NodeGetCapabilities I0215 14:19:49.294697 40638 utils.go:199] ID: 10 GRPC request: {} I0215 14:19:49.294939 40638 utils.go:205] ID: 10 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0215 14:19:49.297687 40638 utils.go:198] ID: 11 GRPC call: /csi.v1.Node/NodeGetCapabilities I0215 14:19:49.297764 40638 utils.go:199] ID: 11 GRPC request: {} I0215 14:19:49.297891 40638 utils.go:205] ID: 11 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0215 14:19:49.298703 40638 utils.go:198] ID: 12 GRPC call: /csi.v1.Node/NodeGetCapabilities I0215 14:19:49.298780 40638 utils.go:199] ID: 12 GRPC request: {} I0215 14:19:49.298916 40638 utils.go:205] ID: 12 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0215 14:19:49.300383 40638 utils.go:198] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC call: /csi.v1.Node/NodePublishVolume I0215 14:19:49.300561 40638 utils.go:199] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount","target_path":"/var/lib/kubelet/pods/51f018b0-29b5-4973-8241-56e5bb7c3cce/volumes/kubernetes.io~csi/pvc-c54a276b-989e-44f7-b7d0-38715293d752/mount","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["discard"]}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"4837cbf8-4f90-4300-b3f6-726c9b9f89b4","imageFeatures":"layering","imageName":"csi-vol-340fa294-d995-4408-932c-94959a1899ac","journalPool":"kube","pool":"kube","storage.kubernetes.io/csiProvisionerIdentity":"1771165005101-2828-rbd.csi.ceph.com"},"volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac"} I0215 14:19:49.300680 40638 nodeserver.go:859] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac target /var/lib/kubelet/pods/51f018b0-29b5-4973-8241-56e5bb7c3cce/volumes/kubernetes.io~csi/pvc-c54a276b-989e-44f7-b7d0-38715293d752/mount isBlock false fstype ext4 stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac readonly false mountflags [bind _netdev discard] I0215 14:19:49.300701 40638 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o bind,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac /var/lib/kubelet/pods/51f018b0-29b5-4973-8241-56e5bb7c3cce/volumes/kubernetes.io~csi/pvc-c54a276b-989e-44f7-b7d0-38715293d752/mount) I0215 14:19:49.311860 40638 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o bind,remount,_netdev,discard /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac /var/lib/kubelet/pods/51f018b0-29b5-4973-8241-56e5bb7c3cce/volumes/kubernetes.io~csi/pvc-c54a276b-989e-44f7-b7d0-38715293d752/mount) I0215 14:19:49.316003 40638 nodeserver.go:742] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac rbd: successfully mounted stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac to targetPath /var/lib/kubelet/pods/51f018b0-29b5-4973-8241-56e5bb7c3cce/volumes/kubernetes.io~csi/pvc-c54a276b-989e-44f7-b7d0-38715293d752/mount I0215 14:19:49.316093 40638 utils.go:205] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC response: {} I0215 14:20:00.124147 40638 utils.go:198] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC call: /csi.v1.Node/NodeUnpublishVolume I0215 14:20:00.124256 40638 utils.go:199] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC request: {"target_path":"/var/lib/kubelet/pods/51f018b0-29b5-4973-8241-56e5bb7c3cce/volumes/kubernetes.io~csi/pvc-c54a276b-989e-44f7-b7d0-38715293d752/mount","volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac"} I0215 14:20:00.124349 40638 mount_linux.go:360] Unmounting /var/lib/kubelet/pods/51f018b0-29b5-4973-8241-56e5bb7c3cce/volumes/kubernetes.io~csi/pvc-c54a276b-989e-44f7-b7d0-38715293d752/mount I0215 14:20:00.137622 40638 nodeserver.go:945] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac rbd: successfully unbound volume 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac from /var/lib/kubelet/pods/51f018b0-29b5-4973-8241-56e5bb7c3cce/volumes/kubernetes.io~csi/pvc-c54a276b-989e-44f7-b7d0-38715293d752/mount I0215 14:20:00.137691 40638 utils.go:205] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC response: {} I0215 14:20:00.229082 40638 utils.go:198] ID: 15 GRPC call: /csi.v1.Node/NodeGetCapabilities I0215 14:20:00.229140 40638 utils.go:199] ID: 15 GRPC request: {} I0215 14:20:00.229224 40638 utils.go:205] ID: 15 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0215 14:20:00.234370 40638 utils.go:198] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC call: /csi.v1.Node/NodeUnstageVolume I0215 14:20:00.234497 40638 utils.go:199] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount","volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac"} I0215 14:20:00.234578 40638 mount_linux.go:360] Unmounting /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac I0215 14:20:00.271127 40638 nodeserver.go:1001] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac successfully unmounted volume (0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac) from staging path (/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/b524d6a911b20fb04a2165455efec849c6c41c7ebf390de1261f8dccad1dd419/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac) I0215 14:20:00.351246 40638 cephcmds.go:105] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac command succeeded: rbd [unmap kube/csi-vol-340fa294-d995-4408-932c-94959a1899ac --device-type krbd --options noudev] I0215 14:20:00.351287 40638 nodeserver.go:1059] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac successfully unmapped volume (0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac) I0215 14:20:00.351446 40638 utils.go:205] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-340fa294-d995-4408-932c-94959a1899ac GRPC response: {}