I0505 16:59:16.484710 42228 cephcsi.go:191] Driver version: v3.11.0 and Git version: bc24b5eca87626d690a29effa9d7420cc0154a7a I0505 16:59:16.484897 42228 cephcsi.go:268] Initial PID limit is set to -1 I0505 16:59:16.484929 42228 cephcsi.go:274] Reconfigured PID limit to -1 (max) I0505 16:59:16.484952 42228 cephcsi.go:223] Starting driver type: rbd with name: rbd.csi.ceph.com I0505 16:59:16.501378 42228 mount_linux.go:282] Detected umount with safe 'not mounted' behavior I0505 16:59:16.524859 42228 rbd_attach.go:242] nbd module loaded I0505 16:59:16.524952 42228 rbd_attach.go:256] kernel version "5.15.0-176-generic" supports cookie feature I0505 16:59:16.557864 42228 rbd_attach.go:272] rbd-nbd tool supports cookie feature I0505 16:59:16.558731 42228 server.go:114] listening for CSI-Addons requests on address: &net.UnixAddr{Name:"/csi/csi-addons.sock", Net:"unix"} I0505 16:59:16.558979 42228 server.go:117] Listening for connections on address: &net.UnixAddr{Name:"//csi/csi.sock", Net:"unix"} I0505 16:59:17.478766 42228 utils.go:198] ID: 1 GRPC call: /csi.v1.Identity/GetPluginInfo I0505 16:59:17.483776 42228 utils.go:199] ID: 1 GRPC request: {} I0505 16:59:17.483853 42228 identityserver-default.go:40] ID: 1 Using default GetPluginInfo I0505 16:59:17.483968 42228 utils.go:205] ID: 1 GRPC response: {"name":"rbd.csi.ceph.com","vendor_version":"v3.11.0"} I0505 16:59:17.539619 42228 utils.go:198] ID: 2 GRPC call: /csi.v1.Node/NodeGetInfo I0505 16:59:17.539701 42228 utils.go:199] ID: 2 GRPC request: {} I0505 16:59:17.539709 42228 nodeserver-default.go:45] ID: 2 Using default NodeGetInfo I0505 16:59:17.539771 42228 utils.go:205] ID: 2 GRPC response: {"accessible_topology":{},"node_id":"instance"} I0505 17:00:17.640719 42228 utils.go:198] ID: 3 GRPC call: /csi.v1.Identity/Probe I0505 17:00:17.640963 42228 utils.go:199] ID: 3 GRPC request: {} I0505 17:00:17.641064 42228 utils.go:205] ID: 3 GRPC response: {} I0505 17:01:17.646637 42228 utils.go:198] ID: 4 GRPC call: /csi.v1.Identity/Probe I0505 17:01:17.646773 42228 utils.go:199] ID: 4 GRPC request: {} I0505 17:01:17.646855 42228 utils.go:205] ID: 4 GRPC response: {} I0505 17:02:17.640392 42228 utils.go:198] ID: 5 GRPC call: /csi.v1.Identity/Probe I0505 17:02:17.640483 42228 utils.go:199] ID: 5 GRPC request: {} I0505 17:02:17.640513 42228 utils.go:205] ID: 5 GRPC response: {} I0505 17:02:35.843604 42228 utils.go:198] ID: 6 GRPC call: /csi.v1.Node/NodeGetCapabilities I0505 17:02:35.843743 42228 utils.go:199] ID: 6 GRPC request: {} I0505 17:02:35.843962 42228 utils.go:205] ID: 6 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0505 17:02:35.850605 42228 utils.go:198] ID: 7 GRPC call: /csi.v1.Node/NodeGetCapabilities I0505 17:02:35.850692 42228 utils.go:199] ID: 7 GRPC request: {} I0505 17:02:35.850821 42228 utils.go:205] ID: 7 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0505 17:02:35.851977 42228 utils.go:198] ID: 8 GRPC call: /csi.v1.Node/NodeGetCapabilities I0505 17:02:35.852198 42228 utils.go:199] ID: 8 GRPC request: {} I0505 17:02:35.852487 42228 utils.go:205] ID: 8 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0505 17:02:35.854343 42228 utils.go:198] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC call: /csi.v1.Node/NodeStageVolume I0505 17:02:35.854950 42228 utils.go:199] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC request: {"secrets":"***stripped***","staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["discard"]}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"4837cbf8-4f90-4300-b3f6-726c9b9f89b4","imageFeatures":"layering","imageName":"csi-vol-e5214765-915d-4351-be29-a96d814deeda","journalPool":"kube","pool":"kube","storage.kubernetes.io/csiProvisionerIdentity":"1778000358658-9989-rbd.csi.ceph.com"},"volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda"} I0505 17:02:35.875142 42228 omap.go:89] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda got omap values: (pool="kube", namespace="", name="csi.volume.e5214765-915d-4351-be29-a96d814deeda"): map[csi.imageid:37d93c35c984 csi.imagename:csi-vol-e5214765-915d-4351-be29-a96d814deeda csi.volname:pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8 csi.volume.owner:default] I0505 17:02:35.923309 42228 rbd_util.go:352] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda checking for ImageFeatures: [layering] I0505 17:02:35.957314 42228 cephcmds.go:105] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda command succeeded: rbd [device list --format=json --device-type krbd] I0505 17:02:35.982750 42228 rbd_attach.go:437] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda rbd: map mon 199.19.213.30 I0505 17:02:36.067489 42228 cephcmds.go:105] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda command succeeded: rbd [--id kube -m 199.19.213.30 --keyfile=***stripped*** map kube/csi-vol-e5214765-915d-4351-be29-a96d814deeda --device-type krbd --options noudev] I0505 17:02:36.067557 42228 nodeserver.go:422] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda rbd image: kube/csi-vol-e5214765-915d-4351-be29-a96d814deeda was successfully mapped at /dev/rbd0 I0505 17:02:36.067676 42228 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0505 17:02:36.078673 42228 mount_linux.go:580] Output: "" I0505 17:02:36.156691 42228 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0505 17:02:36.170171 42228 mount_linux.go:580] Output: "DEVNAME=/dev/rbd0\nTYPE=ext4\n" I0505 17:02:36.170200 42228 mount_linux.go:439] Checking for issues with fsck on disk: /dev/rbd0 I0505 17:02:36.197785 42228 mount_linux.go:544] Attempting to mount disk /dev/rbd0 in ext4 format at /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda I0505 17:02:36.197842 42228 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o _netdev,discard,defaults /dev/rbd0 /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda) I0505 17:02:36.210153 42228 mount_linux.go:577] Attempting to determine if disk "/dev/rbd0" is formatted using blkid with args: ([-p -s TYPE -s PTTYPE -o export /dev/rbd0]) I0505 17:02:36.219723 42228 mount_linux.go:580] Output: "DEVNAME=/dev/rbd0\nTYPE=ext4\n" I0505 17:02:36.219746 42228 resizefs_linux.go:137] ResizeFs.needResize - checking mounted volume /dev/rbd0 I0505 17:02:36.221935 42228 resizefs_linux.go:141] Ext size: filesystem size=5368709120, block size=4096 I0505 17:02:36.221953 42228 resizefs_linux.go:156] Volume /dev/rbd0: device size=5368709120, filesystem size=5368709120, block size=4096 I0505 17:02:36.221985 42228 nodeserver.go:382] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda rbd: successfully mounted volume 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda to stagingTargetPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda I0505 17:02:36.222077 42228 utils.go:205] ID: 9 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC response: {} I0505 17:02:36.223478 42228 utils.go:198] ID: 10 GRPC call: /csi.v1.Node/NodeGetCapabilities I0505 17:02:36.223518 42228 utils.go:199] ID: 10 GRPC request: {} I0505 17:02:36.223617 42228 utils.go:205] ID: 10 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0505 17:02:36.226573 42228 utils.go:198] ID: 11 GRPC call: /csi.v1.Node/NodeGetCapabilities I0505 17:02:36.226610 42228 utils.go:199] ID: 11 GRPC request: {} I0505 17:02:36.226738 42228 utils.go:205] ID: 11 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0505 17:02:36.228147 42228 utils.go:198] ID: 12 GRPC call: /csi.v1.Node/NodeGetCapabilities I0505 17:02:36.228207 42228 utils.go:199] ID: 12 GRPC request: {} I0505 17:02:36.228339 42228 utils.go:205] ID: 12 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0505 17:02:36.229511 42228 utils.go:198] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC call: /csi.v1.Node/NodePublishVolume I0505 17:02:36.229758 42228 utils.go:199] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount","target_path":"/var/lib/kubelet/pods/486f602c-4828-4d75-bf7b-a2b64c2efba0/volumes/kubernetes.io~csi/pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8/mount","volume_capability":{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["discard"]}},"access_mode":{"mode":7}},"volume_context":{"clusterID":"4837cbf8-4f90-4300-b3f6-726c9b9f89b4","imageFeatures":"layering","imageName":"csi-vol-e5214765-915d-4351-be29-a96d814deeda","journalPool":"kube","pool":"kube","storage.kubernetes.io/csiProvisionerIdentity":"1778000358658-9989-rbd.csi.ceph.com"},"volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda"} I0505 17:02:36.229911 42228 nodeserver.go:859] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda target /var/lib/kubelet/pods/486f602c-4828-4d75-bf7b-a2b64c2efba0/volumes/kubernetes.io~csi/pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8/mount isBlock false fstype ext4 stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda readonly false mountflags [bind _netdev discard] I0505 17:02:36.229944 42228 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o bind,_netdev /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda /var/lib/kubelet/pods/486f602c-4828-4d75-bf7b-a2b64c2efba0/volumes/kubernetes.io~csi/pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8/mount) I0505 17:02:36.235581 42228 mount_linux.go:218] Mounting cmd (mount) with arguments (-t ext4 -o bind,remount,_netdev,discard /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda /var/lib/kubelet/pods/486f602c-4828-4d75-bf7b-a2b64c2efba0/volumes/kubernetes.io~csi/pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8/mount) I0505 17:02:36.237261 42228 nodeserver.go:742] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda rbd: successfully mounted stagingPath /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda to targetPath /var/lib/kubelet/pods/486f602c-4828-4d75-bf7b-a2b64c2efba0/volumes/kubernetes.io~csi/pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8/mount I0505 17:02:36.237310 42228 utils.go:205] ID: 13 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC response: {} I0505 17:02:45.610328 42228 utils.go:198] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC call: /csi.v1.Node/NodeUnpublishVolume I0505 17:02:45.610475 42228 utils.go:199] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC request: {"target_path":"/var/lib/kubelet/pods/486f602c-4828-4d75-bf7b-a2b64c2efba0/volumes/kubernetes.io~csi/pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8/mount","volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda"} I0505 17:02:45.610641 42228 mount_linux.go:360] Unmounting /var/lib/kubelet/pods/486f602c-4828-4d75-bf7b-a2b64c2efba0/volumes/kubernetes.io~csi/pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8/mount I0505 17:02:45.615187 42228 nodeserver.go:945] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda rbd: successfully unbound volume 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda from /var/lib/kubelet/pods/486f602c-4828-4d75-bf7b-a2b64c2efba0/volumes/kubernetes.io~csi/pvc-3ae314b6-c906-4543-96b3-9d682e62d8f8/mount I0505 17:02:45.616333 42228 utils.go:205] ID: 14 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC response: {} I0505 17:02:45.714219 42228 utils.go:198] ID: 15 GRPC call: /csi.v1.Node/NodeGetCapabilities I0505 17:02:45.714266 42228 utils.go:199] ID: 15 GRPC request: {} I0505 17:02:45.714377 42228 utils.go:205] ID: 15 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}},{"Type":{"Rpc":{"type":5}}}]} I0505 17:02:45.717589 42228 utils.go:198] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC call: /csi.v1.Node/NodeUnstageVolume I0505 17:02:45.717809 42228 utils.go:199] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC request: {"staging_target_path":"/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount","volume_id":"0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda"} I0505 17:02:45.717921 42228 mount_linux.go:360] Unmounting /var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda I0505 17:02:45.752958 42228 nodeserver.go:1001] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda successfully unmounted volume (0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda) from staging path (/var/lib/kubelet/plugins/kubernetes.io/csi/rbd.csi.ceph.com/4ab8ad71232fa8d9c8054cb6447f32c77298b59db50089af643836577d8006a8/globalmount/0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda) I0505 17:02:45.824384 42228 cephcmds.go:105] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda command succeeded: rbd [unmap kube/csi-vol-e5214765-915d-4351-be29-a96d814deeda --device-type krbd --options noudev] I0505 17:02:45.824433 42228 nodeserver.go:1059] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda successfully unmapped volume (0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda) I0505 17:02:45.824648 42228 utils.go:205] ID: 16 Req-ID: 0001-0024-4837cbf8-4f90-4300-b3f6-726c9b9f89b4-0000000000000002-e5214765-915d-4351-be29-a96d814deeda GRPC response: {}