{"level":"warn","ts":"2026-04-13T02:47:41.387398Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-04-13T02:47:41.387516Z","caller":"etcdmain/etcd.go:73","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.19.213.123:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--experimental-initial-corrupt-check=true","--experimental-watch-progress-notify-interval=5s","--initial-advertise-peer-urls=https://199.19.213.123:2380","--initial-cluster=instance=https://199.19.213.123:2380","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.19.213.123:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.19.213.123:2380","--name=instance","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt"]} {"level":"warn","ts":"2026-04-13T02:47:41.387606Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-04-13T02:47:41.387627Z","caller":"embed/etcd.go:127","msg":"configuring peer listeners","listen-peer-urls":["https://199.19.213.123:2380"]} {"level":"info","ts":"2026-04-13T02:47:41.387687Z","caller":"embed/etcd.go:494","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-04-13T02:47:41.388664Z","caller":"embed/etcd.go:135","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.123:2379"]} {"level":"info","ts":"2026-04-13T02:47:41.388761Z","caller":"embed/etcd.go:308","msg":"starting an etcd server","etcd-version":"3.5.12","git-sha":"e7b3bb6cc","go-version":"go1.20.13","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"instance","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.19.213.123:2380"],"listen-peer-urls":["https://199.19.213.123:2380"],"advertise-client-urls":["https://199.19.213.123:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.123:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"instance=https://199.19.213.123:2380","initial-cluster-state":"new","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"} {"level":"info","ts":"2026-04-13T02:47:41.392468Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"3.438393ms"} {"level":"info","ts":"2026-04-13T02:47:41.397759Z","caller":"etcdserver/raft.go:495","msg":"starting local member","local-member-id":"3e4829a3cabe4fce","cluster-id":"b469cfed817d8605"} {"level":"info","ts":"2026-04-13T02:47:41.399165Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce switched to configuration voters=()"} {"level":"info","ts":"2026-04-13T02:47:41.399204Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce became follower at term 0"} {"level":"info","ts":"2026-04-13T02:47:41.399227Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"newRaft 3e4829a3cabe4fce [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"info","ts":"2026-04-13T02:47:41.399241Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce became follower at term 1"} {"level":"info","ts":"2026-04-13T02:47:41.3993Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce switched to configuration voters=(4487882812132577230)"} {"level":"warn","ts":"2026-04-13T02:47:41.404967Z","caller":"auth/store.go:1241","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-04-13T02:47:41.406363Z","caller":"mvcc/kvstore.go:407","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-04-13T02:47:41.407389Z","caller":"etcdserver/quota.go:94","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-04-13T02:47:41.409259Z","caller":"etcdserver/server.go:860","msg":"starting etcd server","local-member-id":"3e4829a3cabe4fce","local-server-version":"3.5.12","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-04-13T02:47:41.419342Z","caller":"etcdserver/server.go:744","msg":"started as single-node; fast-forwarding election ticks","local-member-id":"3e4829a3cabe4fce","forward-ticks":9,"forward-duration":"900ms","election-ticks":10,"election-timeout":"1s"} {"level":"info","ts":"2026-04-13T02:47:41.419354Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-04-13T02:47:41.419573Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-04-13T02:47:41.419654Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-04-13T02:47:41.420633Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce switched to configuration voters=(4487882812132577230)"} {"level":"info","ts":"2026-04-13T02:47:41.420685Z","caller":"embed/etcd.go:597","msg":"serving peer traffic","address":"199.19.213.123:2380"} {"level":"info","ts":"2026-04-13T02:47:41.42072Z","caller":"embed/etcd.go:569","msg":"cmux::serve","address":"199.19.213.123:2380"} {"level":"info","ts":"2026-04-13T02:47:41.420625Z","caller":"embed/etcd.go:726","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-04-13T02:47:41.420928Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"b469cfed817d8605","local-member-id":"3e4829a3cabe4fce","added-peer-id":"3e4829a3cabe4fce","added-peer-peer-urls":["https://199.19.213.123:2380"]} {"level":"info","ts":"2026-04-13T02:47:41.421321Z","caller":"embed/etcd.go:277","msg":"now serving peer/client/metrics","local-member-id":"3e4829a3cabe4fce","initial-advertise-peer-urls":["https://199.19.213.123:2380"],"listen-peer-urls":["https://199.19.213.123:2380"],"advertise-client-urls":["https://199.19.213.123:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.123:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-04-13T02:47:41.421481Z","caller":"embed/etcd.go:857","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-04-13T02:47:42.300142Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce is starting a new election at term 1"} {"level":"info","ts":"2026-04-13T02:47:42.300232Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce became pre-candidate at term 1"} {"level":"info","ts":"2026-04-13T02:47:42.300259Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce received MsgPreVoteResp from 3e4829a3cabe4fce at term 1"} {"level":"info","ts":"2026-04-13T02:47:42.30028Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce became candidate at term 2"} {"level":"info","ts":"2026-04-13T02:47:42.300291Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce received MsgVoteResp from 3e4829a3cabe4fce at term 2"} {"level":"info","ts":"2026-04-13T02:47:42.300452Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3e4829a3cabe4fce became leader at term 2"} {"level":"info","ts":"2026-04-13T02:47:42.300468Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: 3e4829a3cabe4fce elected leader 3e4829a3cabe4fce at term 2"} {"level":"info","ts":"2026-04-13T02:47:42.30146Z","caller":"etcdserver/server.go:2578","msg":"setting up initial cluster version using v2 API","cluster-version":"3.5"} {"level":"info","ts":"2026-04-13T02:47:42.302332Z","caller":"etcdserver/server.go:2068","msg":"published local member to cluster through raft","local-member-id":"3e4829a3cabe4fce","local-member-attributes":"{Name:instance ClientURLs:[https://199.19.213.123:2379]}","request-path":"/0/members/3e4829a3cabe4fce/attributes","cluster-id":"b469cfed817d8605","publish-timeout":"7s"} {"level":"info","ts":"2026-04-13T02:47:42.302337Z","caller":"embed/serve.go:103","msg":"ready to serve client requests"} {"level":"info","ts":"2026-04-13T02:47:42.302367Z","caller":"embed/serve.go:103","msg":"ready to serve client requests"} {"level":"info","ts":"2026-04-13T02:47:42.302764Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-04-13T02:47:42.302798Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-04-13T02:47:42.303159Z","caller":"membership/cluster.go:584","msg":"set initial cluster version","cluster-id":"b469cfed817d8605","local-member-id":"3e4829a3cabe4fce","cluster-version":"3.5"} {"level":"info","ts":"2026-04-13T02:47:42.30328Z","caller":"api/capability.go:75","msg":"enabled capabilities for version","cluster-version":"3.5"} {"level":"info","ts":"2026-04-13T02:47:42.303322Z","caller":"etcdserver/server.go:2602","msg":"cluster version is updated","cluster-version":"3.5"} {"level":"info","ts":"2026-04-13T02:47:42.306229Z","caller":"embed/serve.go:250","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.19.213.123:2379"} {"level":"info","ts":"2026-04-13T02:47:42.30649Z","caller":"embed/serve.go:250","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"info","ts":"2026-04-13T02:47:44.991281Z","caller":"traceutil/trace.go:171","msg":"trace[200017110] transaction","detail":"{read_only:false; response_revision:94; number_of_response:1; }","duration":"297.916866ms","start":"2026-04-13T02:47:44.693333Z","end":"2026-04-13T02:47:44.99125Z","steps":["trace[200017110] 'process raft request' (duration: 242.572151ms)","trace[200017110] 'compare' (duration: 55.231869ms)"],"step_count":2} {"level":"warn","ts":"2026-04-13T02:47:45.292203Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"162.040248ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:14"} {"level":"info","ts":"2026-04-13T02:47:45.292407Z","caller":"traceutil/trace.go:171","msg":"trace[410570396] transaction","detail":"{read_only:false; response_revision:95; number_of_response:1; }","duration":"295.811076ms","start":"2026-04-13T02:47:44.996583Z","end":"2026-04-13T02:47:45.292394Z","steps":["trace[410570396] 'process raft request' (duration: 133.265562ms)","trace[410570396] 'compare' (duration: 161.914451ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:48:25.86731Z","caller":"traceutil/trace.go:171","msg":"trace[1401629563] transaction","detail":"{read_only:false; response_revision:621; number_of_response:1; }","duration":"154.684491ms","start":"2026-04-13T02:48:25.712601Z","end":"2026-04-13T02:48:25.867285Z","steps":["trace[1401629563] 'process raft request' (duration: 154.531661ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:48:25.876934Z","caller":"traceutil/trace.go:171","msg":"trace[2118144210] transaction","detail":"{read_only:false; response_revision:622; number_of_response:1; }","duration":"162.831475ms","start":"2026-04-13T02:48:25.714078Z","end":"2026-04-13T02:48:25.87691Z","steps":["trace[2118144210] 'process raft request' (duration: 161.680055ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:48:25.87837Z","caller":"traceutil/trace.go:171","msg":"trace[1609250620] transaction","detail":"{read_only:false; response_revision:623; number_of_response:1; }","duration":"160.263058ms","start":"2026-04-13T02:48:25.718084Z","end":"2026-04-13T02:48:25.878347Z","steps":["trace[1609250620] 'process raft request' (duration: 157.923427ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:48:26.52577Z","caller":"traceutil/trace.go:171","msg":"trace[1466027391] transaction","detail":"{read_only:false; response_revision:629; number_of_response:1; }","duration":"120.845743ms","start":"2026-04-13T02:48:26.40489Z","end":"2026-04-13T02:48:26.525736Z","steps":["trace[1466027391] 'process raft request' (duration: 120.202529ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:48:27.980649Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"177.727143ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-04-13T02:48:27.980827Z","caller":"traceutil/trace.go:171","msg":"trace[1097709569] linearizableReadLoop","detail":"{readStateIndex:661; appliedIndex:660; }","duration":"197.214098ms","start":"2026-04-13T02:48:27.783593Z","end":"2026-04-13T02:48:27.980807Z","steps":["trace[1097709569] 'read index received' (duration: 19.253619ms)","trace[1097709569] 'applied index is now lower than readState.Index' (duration: 177.958749ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:48:27.980919Z","caller":"traceutil/trace.go:171","msg":"trace[1076659495] transaction","detail":"{read_only:false; response_revision:636; number_of_response:1; }","duration":"434.036241ms","start":"2026-04-13T02:48:27.546842Z","end":"2026-04-13T02:48:27.980878Z","steps":["trace[1076659495] 'process raft request' (duration: 256.012397ms)","trace[1076659495] 'compare' (duration: 177.426953ms)"],"step_count":2} {"level":"warn","ts":"2026-04-13T02:48:27.981046Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"110.450192ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:434"} {"level":"info","ts":"2026-04-13T02:48:27.981082Z","caller":"traceutil/trace.go:171","msg":"trace[689523633] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:636; }","duration":"110.523757ms","start":"2026-04-13T02:48:27.870547Z","end":"2026-04-13T02:48:27.981071Z","steps":["trace[689523633] 'agreement among raft nodes before linearized reading' (duration: 110.354746ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:48:27.981117Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"197.501329ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-tzdqd\" ","response":"range_response_count:1 size:16965"} {"level":"warn","ts":"2026-04-13T02:48:27.9811Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T02:48:27.546813Z","time spent":"434.193462ms","remote":"127.0.0.1:54036","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":1627,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-04-13T02:48:27.981199Z","caller":"traceutil/trace.go:171","msg":"trace[1302240322] range","detail":"{range_begin:/registry/pods/kube-system/cilium-tzdqd; range_end:; response_count:1; response_revision:636; }","duration":"197.615817ms","start":"2026-04-13T02:48:27.783562Z","end":"2026-04-13T02:48:27.981177Z","steps":["trace[1302240322] 'agreement among raft nodes before linearized reading' (duration: 197.322317ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:50:05.707881Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"135.31499ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" ","response":"range_response_count:1 size:5412"} {"level":"info","ts":"2026-04-13T02:50:05.707992Z","caller":"traceutil/trace.go:171","msg":"trace[804903946] range","detail":"{range_begin:/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb; range_end:; response_count:1; response_revision:1903; }","duration":"135.459575ms","start":"2026-04-13T02:50:05.572505Z","end":"2026-04-13T02:50:05.707965Z","steps":["trace[804903946] 'range keys from in-memory index tree' (duration: 135.122874ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:50:05.708346Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"152.968496ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/local-path-storage/helper-pod-create-pvc-5ac59fba-0730-42cb-a5ed-fc24be6c35dc\" ","response":"range_response_count:1 size:3627"} {"level":"info","ts":"2026-04-13T02:50:05.708441Z","caller":"traceutil/trace.go:171","msg":"trace[134539805] range","detail":"{range_begin:/registry/pods/local-path-storage/helper-pod-create-pvc-5ac59fba-0730-42cb-a5ed-fc24be6c35dc; range_end:; response_count:1; response_revision:1903; }","duration":"153.10931ms","start":"2026-04-13T02:50:05.555312Z","end":"2026-04-13T02:50:05.708421Z","steps":["trace[134539805] 'range keys from in-memory index tree' (duration: 152.843652ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:50:36.039105Z","caller":"traceutil/trace.go:171","msg":"trace[1480981502] linearizableReadLoop","detail":"{readStateIndex:2198; appliedIndex:2197; }","duration":"137.926589ms","start":"2026-04-13T02:50:35.901156Z","end":"2026-04-13T02:50:36.039083Z","steps":["trace[1480981502] 'read index received' (duration: 100.434505ms)","trace[1480981502] 'applied index is now lower than readState.Index' (duration: 37.490944ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:50:36.039221Z","caller":"traceutil/trace.go:171","msg":"trace[2064669774] transaction","detail":"{read_only:false; response_revision:2118; number_of_response:1; }","duration":"138.283856ms","start":"2026-04-13T02:50:35.900918Z","end":"2026-04-13T02:50:36.039202Z","steps":["trace[2064669774] 'process raft request' (duration: 100.685666ms)","trace[2064669774] 'compare' (duration: 37.37679ms)"],"step_count":2} {"level":"warn","ts":"2026-04-13T02:50:36.039335Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"138.151399ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-04-13T02:50:36.03938Z","caller":"traceutil/trace.go:171","msg":"trace[53219972] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:1; response_revision:2118; }","duration":"138.232933ms","start":"2026-04-13T02:50:35.901135Z","end":"2026-04-13T02:50:36.039368Z","steps":["trace[53219972] 'agreement among raft nodes before linearized reading' (duration: 138.075016ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:50:36.039504Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"135.887598ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-04-13T02:50:36.039536Z","caller":"traceutil/trace.go:171","msg":"trace[2127329389] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:2118; }","duration":"135.942741ms","start":"2026-04-13T02:50:35.903583Z","end":"2026-04-13T02:50:36.039526Z","steps":["trace[2127329389] 'agreement among raft nodes before linearized reading' (duration: 135.889159ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:50:36.03982Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"130.484878ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" ","response":"range_response_count:1 size:5412"} {"level":"info","ts":"2026-04-13T02:50:36.03987Z","caller":"traceutil/trace.go:171","msg":"trace[1807783735] range","detail":"{range_begin:/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb; range_end:; response_count:1; response_revision:2118; }","duration":"130.557011ms","start":"2026-04-13T02:50:35.909298Z","end":"2026-04-13T02:50:36.039855Z","steps":["trace[1807783735] 'agreement among raft nodes before linearized reading' (duration: 130.371522ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:50:36.222115Z","caller":"traceutil/trace.go:171","msg":"trace[1636426465] transaction","detail":"{read_only:false; response_revision:2119; number_of_response:1; }","duration":"176.833298ms","start":"2026-04-13T02:50:36.045244Z","end":"2026-04-13T02:50:36.222078Z","steps":["trace[1636426465] 'process raft request' (duration: 140.237793ms)","trace[1636426465] 'compare' (duration: 36.094592ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:50:37.27546Z","caller":"traceutil/trace.go:171","msg":"trace[1373631482] transaction","detail":"{read_only:false; response_revision:2129; number_of_response:1; }","duration":"200.944411ms","start":"2026-04-13T02:50:37.074507Z","end":"2026-04-13T02:50:37.275451Z","steps":["trace[1373631482] 'process raft request' (duration: 200.91616ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:50:37.275566Z","caller":"traceutil/trace.go:171","msg":"trace[1381787339] transaction","detail":"{read_only:false; response_revision:2128; number_of_response:1; }","duration":"201.617081ms","start":"2026-04-13T02:50:37.073944Z","end":"2026-04-13T02:50:37.275561Z","steps":["trace[1381787339] 'process raft request' (duration: 201.453954ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:50:37.275459Z","caller":"traceutil/trace.go:171","msg":"trace[766188273] transaction","detail":"{read_only:false; response_revision:2127; number_of_response:1; }","duration":"203.99762ms","start":"2026-04-13T02:50:37.071442Z","end":"2026-04-13T02:50:37.275439Z","steps":["trace[766188273] 'process raft request' (duration: 125.7527ms)","trace[766188273] 'compare' (duration: 78.114644ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:55:59.172858Z","caller":"traceutil/trace.go:171","msg":"trace[1961585632] transaction","detail":"{read_only:false; response_revision:4670; number_of_response:1; }","duration":"180.785087ms","start":"2026-04-13T02:55:58.992044Z","end":"2026-04-13T02:55:59.172829Z","steps":["trace[1961585632] 'process raft request' (duration: 180.500418ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:56:06.131133Z","caller":"traceutil/trace.go:171","msg":"trace[900508101] linearizableReadLoop","detail":"{readStateIndex:4900; appliedIndex:4899; }","duration":"194.889448ms","start":"2026-04-13T02:56:05.936219Z","end":"2026-04-13T02:56:06.131109Z","steps":["trace[900508101] 'read index received' (duration: 194.732602ms)","trace[900508101] 'applied index is now lower than readState.Index' (duration: 155.796µs)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:56:06.131212Z","caller":"traceutil/trace.go:171","msg":"trace[1566603344] transaction","detail":"{read_only:false; response_revision:4724; number_of_response:1; }","duration":"196.581005ms","start":"2026-04-13T02:56:05.934592Z","end":"2026-04-13T02:56:06.131173Z","steps":["trace[1566603344] 'process raft request' (duration: 196.315156ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:56:06.131395Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"194.63781ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/monitoring/kube-prometheus-stack-grafana-6d64d7985d-zks4t.18a5cb1fa830f552\" ","response":"range_response_count:1 size:898"} {"level":"info","ts":"2026-04-13T02:56:06.131442Z","caller":"traceutil/trace.go:171","msg":"trace[2043419562] range","detail":"{range_begin:/registry/events/monitoring/kube-prometheus-stack-grafana-6d64d7985d-zks4t.18a5cb1fa830f552; range_end:; response_count:1; response_revision:4724; }","duration":"194.707472ms","start":"2026-04-13T02:56:05.936721Z","end":"2026-04-13T02:56:06.131428Z","steps":["trace[2043419562] 'agreement among raft nodes before linearized reading' (duration: 194.544947ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:56:06.131764Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"195.519179ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumes/pvc-cc919132-2cbb-4e4e-a7ee-e4217528c1f9\" ","response":"range_response_count:1 size:1450"} {"level":"info","ts":"2026-04-13T02:56:06.131839Z","caller":"traceutil/trace.go:171","msg":"trace[1152117559] range","detail":"{range_begin:/registry/persistentvolumes/pvc-cc919132-2cbb-4e4e-a7ee-e4217528c1f9; range_end:; response_count:1; response_revision:4724; }","duration":"195.624413ms","start":"2026-04-13T02:56:05.936196Z","end":"2026-04-13T02:56:06.13182Z","steps":["trace[1152117559] 'agreement among raft nodes before linearized reading' (duration: 195.018572ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:56:06.265864Z","caller":"traceutil/trace.go:171","msg":"trace[1143089239] transaction","detail":"{read_only:false; response_revision:4730; number_of_response:1; }","duration":"113.77131ms","start":"2026-04-13T02:56:06.152056Z","end":"2026-04-13T02:56:06.265827Z","steps":["trace[1143089239] 'process raft request' (duration: 91.646495ms)","trace[1143089239] 'compare' (duration: 21.844214ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:56:11.864554Z","caller":"traceutil/trace.go:171","msg":"trace[1885046102] transaction","detail":"{read_only:false; response_revision:4844; number_of_response:1; }","duration":"113.152454ms","start":"2026-04-13T02:56:11.75134Z","end":"2026-04-13T02:56:11.864493Z","steps":["trace[1885046102] 'process raft request' (duration: 112.984899ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:57:33.321097Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"338.918182ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-04-13T02:57:33.321256Z","caller":"traceutil/trace.go:171","msg":"trace[207208556] linearizableReadLoop","detail":"{readStateIndex:5762; appliedIndex:5761; }","duration":"485.552318ms","start":"2026-04-13T02:57:32.835683Z","end":"2026-04-13T02:57:33.321235Z","steps":["trace[207208556] 'read index received' (duration: 146.167526ms)","trace[207208556] 'applied index is now lower than readState.Index' (duration: 339.382762ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:57:33.321303Z","caller":"traceutil/trace.go:171","msg":"trace[191841193] transaction","detail":"{read_only:false; response_revision:5563; number_of_response:1; }","duration":"545.504986ms","start":"2026-04-13T02:57:32.775755Z","end":"2026-04-13T02:57:33.32126Z","steps":["trace[191841193] 'process raft request' (duration: 206.107153ms)","trace[191841193] 'compare' (duration: 338.652171ms)"],"step_count":2} {"level":"warn","ts":"2026-04-13T02:57:33.321579Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"485.880503ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" ","response":"range_response_count:1 size:439"} {"level":"info","ts":"2026-04-13T02:57:33.321708Z","caller":"traceutil/trace.go:171","msg":"trace[1133811142] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:5563; }","duration":"486.0591ms","start":"2026-04-13T02:57:32.835627Z","end":"2026-04-13T02:57:33.321686Z","steps":["trace[1133811142] 'agreement among raft nodes before linearized reading' (duration: 485.705734ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:57:33.321837Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T02:57:32.835603Z","time spent":"486.219927ms","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":462,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" "} {"level":"warn","ts":"2026-04-13T02:57:33.322246Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"281.236981ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattachments/\" range_end:\"/registry/volumeattachments0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-04-13T02:57:33.322345Z","caller":"traceutil/trace.go:171","msg":"trace[239056859] range","detail":"{range_begin:/registry/volumeattachments/; range_end:/registry/volumeattachments0; response_count:0; response_revision:5563; }","duration":"281.357127ms","start":"2026-04-13T02:57:33.040975Z","end":"2026-04-13T02:57:33.322332Z","steps":["trace[239056859] 'agreement among raft nodes before linearized reading' (duration: 281.220351ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:57:33.322461Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T02:57:32.775726Z","time spent":"545.738286ms","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":510,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-04-13T02:57:33.322719Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"136.967067ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" ","response":"range_response_count:1 size:555"} {"level":"info","ts":"2026-04-13T02:57:33.3228Z","caller":"traceutil/trace.go:171","msg":"trace[601824679] range","detail":"{range_begin:/registry/leases/openstack/rabbitmq-cluster-operator-leader-election; range_end:; response_count:1; response_revision:5563; }","duration":"137.146624ms","start":"2026-04-13T02:57:33.185639Z","end":"2026-04-13T02:57:33.322785Z","steps":["trace[601824679] 'agreement among raft nodes before linearized reading' (duration: 136.976717ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:57:33.323115Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"295.396099ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" ","response":"range_response_count:1 size:478"} {"level":"info","ts":"2026-04-13T02:57:33.323179Z","caller":"traceutil/trace.go:171","msg":"trace[1440093862] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:5563; }","duration":"295.525965ms","start":"2026-04-13T02:57:33.027639Z","end":"2026-04-13T02:57:33.323165Z","steps":["trace[1440093862] 'agreement among raft nodes before linearized reading' (duration: 295.342117ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:57:33.323275Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"111.749918ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/jobs/openstack/keystone-db-sync\" ","response":"range_response_count:1 size:8317"} {"level":"info","ts":"2026-04-13T02:57:33.323394Z","caller":"traceutil/trace.go:171","msg":"trace[1131468269] range","detail":"{range_begin:/registry/jobs/openstack/keystone-db-sync; range_end:; response_count:1; response_revision:5563; }","duration":"111.936087ms","start":"2026-04-13T02:57:33.211436Z","end":"2026-04-13T02:57:33.323372Z","steps":["trace[1131468269] 'agreement among raft nodes before linearized reading' (duration: 109.940289ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:57:33.323702Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"468.250926ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:435"} {"level":"info","ts":"2026-04-13T02:57:33.323759Z","caller":"traceutil/trace.go:171","msg":"trace[1067844545] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:5563; }","duration":"468.349301ms","start":"2026-04-13T02:57:32.855393Z","end":"2026-04-13T02:57:33.323742Z","steps":["trace[1067844545] 'agreement among raft nodes before linearized reading' (duration: 465.946126ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:57:33.323799Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T02:57:32.855373Z","time spent":"468.416774ms","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Range","request count":0,"request size":44,"response count":1,"response size":458,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" "} {"level":"warn","ts":"2026-04-13T02:57:33.609069Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"156.840873ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-04-13T02:57:33.60922Z","caller":"traceutil/trace.go:171","msg":"trace[1008689943] transaction","detail":"{read_only:false; response_revision:5564; number_of_response:1; }","duration":"281.117086ms","start":"2026-04-13T02:57:33.328076Z","end":"2026-04-13T02:57:33.609193Z","steps":["trace[1008689943] 'process raft request' (duration: 123.990102ms)","trace[1008689943] 'compare' (duration: 156.585861ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:57:33.620423Z","caller":"traceutil/trace.go:171","msg":"trace[409058039] transaction","detail":"{read_only:false; response_revision:5565; number_of_response:1; }","duration":"291.924967ms","start":"2026-04-13T02:57:33.328467Z","end":"2026-04-13T02:57:33.620392Z","steps":["trace[409058039] 'process raft request' (duration: 291.473258ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:57:33.620933Z","caller":"traceutil/trace.go:171","msg":"trace[124044437] transaction","detail":"{read_only:false; response_revision:5566; number_of_response:1; }","duration":"292.388136ms","start":"2026-04-13T02:57:33.328516Z","end":"2026-04-13T02:57:33.620904Z","steps":["trace[124044437] 'process raft request' (duration: 291.724638ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:57:33.62111Z","caller":"traceutil/trace.go:171","msg":"trace[209601512] linearizableReadLoop","detail":"{readStateIndex:5766; appliedIndex:5762; }","duration":"240.308369ms","start":"2026-04-13T02:57:33.380787Z","end":"2026-04-13T02:57:33.621095Z","steps":["trace[209601512] 'read index received' (duration: 71.395881ms)","trace[209601512] 'applied index is now lower than readState.Index' (duration: 168.911158ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T02:57:33.621365Z","caller":"traceutil/trace.go:171","msg":"trace[296476198] transaction","detail":"{read_only:false; response_revision:5567; number_of_response:1; }","duration":"290.125328ms","start":"2026-04-13T02:57:33.331228Z","end":"2026-04-13T02:57:33.621353Z","steps":["trace[296476198] 'process raft request' (duration: 289.591375ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T02:57:33.622631Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"241.693809ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rabbitmq.com/federations/\" range_end:\"/registry/rabbitmq.com/federations0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-04-13T02:57:33.622697Z","caller":"traceutil/trace.go:171","msg":"trace[530438392] range","detail":"{range_begin:/registry/rabbitmq.com/federations/; range_end:/registry/rabbitmq.com/federations0; response_count:0; response_revision:5567; }","duration":"241.93631ms","start":"2026-04-13T02:57:33.380744Z","end":"2026-04-13T02:57:33.622681Z","steps":["trace[530438392] 'agreement among raft nodes before linearized reading' (duration: 241.70255ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T02:57:42.340242Z","caller":"mvcc/index.go:214","msg":"compact tree index","revision":2904} {"level":"info","ts":"2026-04-13T02:57:42.433953Z","caller":"mvcc/kvstore_compaction.go:68","msg":"finished scheduled compaction","compact-revision":2904,"took":"92.520804ms","hash":3298566780,"current-db-size-bytes":54206464,"current-db-size":"54 MB","current-db-size-in-use-bytes":29224960,"current-db-size-in-use":"29 MB"} {"level":"info","ts":"2026-04-13T02:57:42.434022Z","caller":"mvcc/hash.go:137","msg":"storing new hash","hash":3298566780,"revision":2904,"compact-revision":-1} {"level":"warn","ts":"2026-04-13T03:00:20.823395Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"118.163981ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterrolebindings/\" range_end:\"/registry/clusterrolebindings0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-04-13T03:00:20.823475Z","caller":"traceutil/trace.go:171","msg":"trace[1740294578] range","detail":"{range_begin:/registry/clusterrolebindings/; range_end:/registry/clusterrolebindings0; response_count:0; response_revision:6721; }","duration":"118.286437ms","start":"2026-04-13T03:00:20.705171Z","end":"2026-04-13T03:00:20.823457Z","steps":["trace[1740294578] 'count revisions from in-memory index tree' (duration: 118.098777ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T03:00:26.683084Z","caller":"traceutil/trace.go:171","msg":"trace[1684278284] transaction","detail":"{read_only:false; response_revision:6749; number_of_response:1; }","duration":"140.850925ms","start":"2026-04-13T03:00:26.542206Z","end":"2026-04-13T03:00:26.683056Z","steps":["trace[1684278284] 'process raft request' (duration: 140.721528ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T03:00:26.88208Z","caller":"traceutil/trace.go:171","msg":"trace[635685601] linearizableReadLoop","detail":"{readStateIndex:6995; appliedIndex:6993; }","duration":"294.63735ms","start":"2026-04-13T03:00:26.587414Z","end":"2026-04-13T03:00:26.882051Z","steps":["trace[635685601] 'read index received' (duration: 95.714169ms)","trace[635685601] 'applied index is now lower than readState.Index' (duration: 198.922301ms)"],"step_count":2} {"level":"warn","ts":"2026-04-13T03:00:26.882505Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"295.072582ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:435"} {"level":"info","ts":"2026-04-13T03:00:26.882535Z","caller":"traceutil/trace.go:171","msg":"trace[1073020391] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:6753; }","duration":"295.164367ms","start":"2026-04-13T03:00:26.587362Z","end":"2026-04-13T03:00:26.882526Z","steps":["trace[1073020391] 'agreement among raft nodes before linearized reading' (duration: 295.02271ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:00:26.882694Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"219.647499ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/gateway.networking.k8s.io/grpcroutes/\" range_end:\"/registry/gateway.networking.k8s.io/grpcroutes0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-04-13T03:00:26.882717Z","caller":"traceutil/trace.go:171","msg":"trace[1514688857] range","detail":"{range_begin:/registry/gateway.networking.k8s.io/grpcroutes/; range_end:/registry/gateway.networking.k8s.io/grpcroutes0; response_count:0; response_revision:6753; }","duration":"219.708902ms","start":"2026-04-13T03:00:26.663Z","end":"2026-04-13T03:00:26.882709Z","steps":["trace[1514688857] 'agreement among raft nodes before linearized reading' (duration: 219.66882ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:00:26.882883Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"132.414374ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumeclaims/openstack/persistence-rabbitmq-keystone-server-0\" ","response":"range_response_count:1 size:1865"} {"level":"info","ts":"2026-04-13T03:00:26.882905Z","caller":"traceutil/trace.go:171","msg":"trace[1727967616] range","detail":"{range_begin:/registry/persistentvolumeclaims/openstack/persistence-rabbitmq-keystone-server-0; range_end:; response_count:1; response_revision:6753; }","duration":"132.436825ms","start":"2026-04-13T03:00:26.750462Z","end":"2026-04-13T03:00:26.882898Z","steps":["trace[1727967616] 'agreement among raft nodes before linearized reading' (duration: 132.372312ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T03:00:26.883129Z","caller":"traceutil/trace.go:171","msg":"trace[338504321] transaction","detail":"{read_only:false; response_revision:6750; number_of_response:1; }","duration":"339.907492ms","start":"2026-04-13T03:00:26.543192Z","end":"2026-04-13T03:00:26.883099Z","steps":["trace[338504321] 'process raft request' (duration: 338.599846ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:00:26.883855Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:26.543183Z","time spent":"340.597898ms","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":463,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-04-13T03:00:26.883274Z","caller":"traceutil/trace.go:171","msg":"trace[825618977] transaction","detail":"{read_only:false; response_revision:6752; number_of_response:1; }","duration":"238.127973ms","start":"2026-04-13T03:00:26.645128Z","end":"2026-04-13T03:00:26.883256Z","steps":["trace[825618977] 'process raft request' (duration: 236.864018ms)"],"step_count":1} {"level":"info","ts":"2026-04-13T03:00:26.884589Z","caller":"traceutil/trace.go:171","msg":"trace[762373424] transaction","detail":"{read_only:false; response_revision:6751; number_of_response:1; }","duration":"338.330991ms","start":"2026-04-13T03:00:26.544752Z","end":"2026-04-13T03:00:26.883083Z","steps":["trace[762373424] 'process raft request' (duration: 337.204344ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:00:26.884638Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:26.544715Z","time spent":"339.898791ms","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-04-13T03:00:26.884881Z","caller":"traceutil/trace.go:171","msg":"trace[473298140] transaction","detail":"{read_only:false; response_revision:6753; number_of_response:1; }","duration":"232.990591ms","start":"2026-04-13T03:00:26.649301Z","end":"2026-04-13T03:00:26.882292Z","steps":["trace[473298140] 'process raft request' (duration: 232.722688ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:00:59.405609Z","caller":"etcdserver/v3_server.go:897","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5750706967628126102,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-04-13T03:00:59.906057Z","caller":"etcdserver/v3_server.go:897","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5750706967628126102,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-04-13T03:01:00.406882Z","caller":"etcdserver/v3_server.go:897","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5750706967628126102,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-04-13T03:01:00.905294Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000494845s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-04-13T03:01:00.905371Z","caller":"traceutil/trace.go:171","msg":"trace[1282030834] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000608141s","start":"2026-04-13T03:00:58.904744Z","end":"2026-04-13T03:01:00.905352Z","steps":["trace[1282030834] 'agreement among raft nodes before linearized reading' (duration: 2.000520657s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:00.905551Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:58.904727Z","time spent":"2.000810962s","remote":"127.0.0.1:53558","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/04/13 03:01:00 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-04-13T03:01:00.907822Z","caller":"etcdserver/v3_server.go:897","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5750706967628126102,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-04-13T03:01:01.135151Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"2.294541164s","expected-duration":"1s"} {"level":"info","ts":"2026-04-13T03:01:01.135531Z","caller":"traceutil/trace.go:171","msg":"trace[636832720] transaction","detail":"{read_only:false; response_revision:6936; number_of_response:1; }","duration":"2.294976637s","start":"2026-04-13T03:00:58.840532Z","end":"2026-04-13T03:01:01.135508Z","steps":["trace[636832720] 'process raft request' (duration: 2.29484004s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.135656Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:58.84052Z","time spent":"2.295059191s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":540,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-04-13T03:01:01.273103Z","caller":"traceutil/trace.go:171","msg":"trace[1436366299] transaction","detail":"{read_only:false; response_revision:6938; number_of_response:1; }","duration":"2.270964382s","start":"2026-04-13T03:00:59.002109Z","end":"2026-04-13T03:01:01.273073Z","steps":["trace[1436366299] 'process raft request' (duration: 2.270673047s)"],"step_count":1} {"level":"info","ts":"2026-04-13T03:01:01.273167Z","caller":"traceutil/trace.go:171","msg":"trace[1288713178] transaction","detail":"{read_only:false; response_revision:6937; number_of_response:1; }","duration":"2.272116972s","start":"2026-04-13T03:00:59.001029Z","end":"2026-04-13T03:01:01.273146Z","steps":["trace[1288713178] 'process raft request' (duration: 2.266820128s)"],"step_count":1} {"level":"info","ts":"2026-04-13T03:01:01.273184Z","caller":"traceutil/trace.go:171","msg":"trace[1023445206] transaction","detail":"{read_only:false; response_revision:6939; number_of_response:1; }","duration":"1.830810338s","start":"2026-04-13T03:00:59.442336Z","end":"2026-04-13T03:01:01.273147Z","steps":["trace[1023445206] 'process raft request' (duration: 1.830533614s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.273276Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.001008Z","time spent":"2.272204697s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":555,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-04-13T03:01:01.273242Z","caller":"traceutil/trace.go:171","msg":"trace[394737214] transaction","detail":"{read_only:false; response_revision:6940; number_of_response:1; }","duration":"1.09376594s","start":"2026-04-13T03:01:00.179438Z","end":"2026-04-13T03:01:01.273204Z","steps":["trace[394737214] 'process raft request' (duration: 1.093467845s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.273336Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.002102Z","time spent":"2.271076248s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":535,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-04-13T03:01:01.273345Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.442316Z","time spent":"1.830943145s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":673,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-04-13T03:01:01.273544Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:01:00.179416Z","time spent":"1.093986571s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":509,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-04-13T03:01:01.287347Z","caller":"traceutil/trace.go:171","msg":"trace[1649879850] linearizableReadLoop","detail":"{readStateIndex:7191; appliedIndex:7185; }","duration":"2.382548227s","start":"2026-04-13T03:00:58.904785Z","end":"2026-04-13T03:01:01.287333Z","steps":["trace[1649879850] 'read index received' (duration: 2.230670335s)","trace[1649879850] 'applied index is now lower than readState.Index' (duration: 151.877102ms)"],"step_count":2} {"level":"info","ts":"2026-04-13T03:01:01.287391Z","caller":"traceutil/trace.go:171","msg":"trace[191640558] transaction","detail":"{read_only:false; response_revision:6941; number_of_response:1; }","duration":"221.507503ms","start":"2026-04-13T03:01:01.065863Z","end":"2026-04-13T03:01:01.287371Z","steps":["trace[191640558] 'process raft request' (duration: 221.220999ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.28746Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"381.704594ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-04-13T03:01:01.287485Z","caller":"traceutil/trace.go:171","msg":"trace[42266720] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:6941; }","duration":"381.768618ms","start":"2026-04-13T03:01:00.905709Z","end":"2026-04-13T03:01:01.287478Z","steps":["trace[42266720] 'agreement among raft nodes before linearized reading' (duration: 381.716455ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.287521Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:01:00.905692Z","time spent":"381.8097ms","remote":"127.0.0.1:53560","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-04-13T03:01:01.287517Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.254916644s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" ","response":"range_response_count:1 size:478"} {"level":"info","ts":"2026-04-13T03:01:01.287575Z","caller":"traceutil/trace.go:171","msg":"trace[2064255610] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:6941; }","duration":"2.255003488s","start":"2026-04-13T03:00:59.032554Z","end":"2026-04-13T03:01:01.287558Z","steps":["trace[2064255610] 'agreement among raft nodes before linearized reading' (duration: 2.254873181s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.287619Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.032537Z","time spent":"2.255074672s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Range","request count":0,"request size":45,"response count":1,"response size":501,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" "} {"level":"warn","ts":"2026-04-13T03:01:01.287678Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"225.826276ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumpodippools/\" range_end:\"/registry/cilium.io/ciliumpodippools0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-04-13T03:01:01.287709Z","caller":"traceutil/trace.go:171","msg":"trace[954429329] range","detail":"{range_begin:/registry/cilium.io/ciliumpodippools/; range_end:/registry/cilium.io/ciliumpodippools0; response_count:0; response_revision:6941; }","duration":"225.895999ms","start":"2026-04-13T03:01:01.061804Z","end":"2026-04-13T03:01:01.2877Z","steps":["trace[954429329] 'agreement among raft nodes before linearized reading' (duration: 225.846967ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.287861Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.119961719s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets/monitoring/kube-prometheus-stack-prometheus-client-secret\" ","response":"range_response_count:1 size:692"} {"level":"warn","ts":"2026-04-13T03:01:01.287872Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.995800464s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" ","response":"range_response_count:1 size:492"} {"level":"info","ts":"2026-04-13T03:01:01.287905Z","caller":"traceutil/trace.go:171","msg":"trace[722508773] range","detail":"{range_begin:/registry/secrets/monitoring/kube-prometheus-stack-prometheus-client-secret; range_end:; response_count:1; response_revision:6941; }","duration":"2.120027403s","start":"2026-04-13T03:00:59.167864Z","end":"2026-04-13T03:01:01.287891Z","steps":["trace[722508773] 'agreement among raft nodes before linearized reading' (duration: 2.119927268s)"],"step_count":1} {"level":"info","ts":"2026-04-13T03:01:01.287919Z","caller":"traceutil/trace.go:171","msg":"trace[793903178] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:6941; }","duration":"1.99589714s","start":"2026-04-13T03:00:59.292007Z","end":"2026-04-13T03:01:01.287905Z","steps":["trace[793903178] 'agreement among raft nodes before linearized reading' (duration: 1.995702059s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.287942Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.167853Z","time spent":"2.120079796s","remote":"127.0.0.1:53648","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":715,"request content":"key:\"/registry/secrets/monitoring/kube-prometheus-stack-prometheus-client-secret\" "} {"level":"warn","ts":"2026-04-13T03:01:01.28796Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.29199Z","time spent":"1.995960393s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Range","request count":0,"request size":60,"response count":1,"response size":515,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" "} {"level":"warn","ts":"2026-04-13T03:01:01.288021Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.430348767s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" ","response":"range_response_count:1 size:507"} {"level":"info","ts":"2026-04-13T03:01:01.28805Z","caller":"traceutil/trace.go:171","msg":"trace[1896372773] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:6941; }","duration":"1.43041539s","start":"2026-04-13T03:00:59.857626Z","end":"2026-04-13T03:01:01.288041Z","steps":["trace[1896372773] 'agreement among raft nodes before linearized reading' (duration: 1.430317645s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.288076Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.857608Z","time spent":"1.430461462s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Range","request count":0,"request size":54,"response count":1,"response size":530,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" "} {"level":"warn","ts":"2026-04-13T03:01:01.288183Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.233359341s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:434"} {"level":"info","ts":"2026-04-13T03:01:01.288222Z","caller":"traceutil/trace.go:171","msg":"trace[683880749] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:6941; }","duration":"2.233418544s","start":"2026-04-13T03:00:59.054795Z","end":"2026-04-13T03:01:01.288213Z","steps":["trace[683880749] 'agreement among raft nodes before linearized reading' (duration: 2.233285777s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.288241Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.251523659s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" ","response":"range_response_count:1 size:438"} {"level":"warn","ts":"2026-04-13T03:01:01.28825Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.054784Z","time spent":"2.233460196s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Range","request count":0,"request size":44,"response count":1,"response size":457,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" "} {"level":"info","ts":"2026-04-13T03:01:01.288276Z","caller":"traceutil/trace.go:171","msg":"trace[1930021218] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:6941; }","duration":"2.251589583s","start":"2026-04-13T03:00:59.036678Z","end":"2026-04-13T03:01:01.288267Z","steps":["trace[1930021218] 'agreement among raft nodes before linearized reading' (duration: 2.251500928s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.288306Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:00:59.036662Z","time spent":"2.251638635s","remote":"127.0.0.1:53836","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" "} {"level":"warn","ts":"2026-04-13T03:01:01.28817Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.258470555s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" ","response":"range_response_count:1 size:12215"} {"level":"info","ts":"2026-04-13T03:01:01.290202Z","caller":"traceutil/trace.go:171","msg":"trace[1691915445] range","detail":"{range_begin:/registry/pods/openstack/percona-xtradb-pxc-0; range_end:; response_count:1; response_revision:6941; }","duration":"1.260576733s","start":"2026-04-13T03:01:00.029608Z","end":"2026-04-13T03:01:01.290184Z","steps":["trace[1691915445] 'agreement among raft nodes before linearized reading' (duration: 1.258289606s)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.290256Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:01:00.029597Z","time spent":"1.260642886s","remote":"127.0.0.1:53764","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":12238,"request content":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" "} {"level":"warn","ts":"2026-04-13T03:01:01.292368Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"761.550878ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs/\" range_end:\"/registry/services/specs0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-04-13T03:01:01.292663Z","caller":"traceutil/trace.go:171","msg":"trace[580643187] range","detail":"{range_begin:/registry/services/specs/; range_end:/registry/services/specs0; response_count:0; response_revision:6941; }","duration":"761.884836ms","start":"2026-04-13T03:01:00.530766Z","end":"2026-04-13T03:01:01.292651Z","steps":["trace[580643187] 'agreement among raft nodes before linearized reading' (duration: 757.544402ms)"],"step_count":1} {"level":"warn","ts":"2026-04-13T03:01:01.292704Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-04-13T03:01:00.530749Z","time spent":"761.943299ms","remote":"127.0.0.1:53766","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":40,"response size":30,"request content":"key:\"/registry/services/specs/\" range_end:\"/registry/services/specs0\" count_only:true "} {"level":"info","ts":"2026-04-13T03:02:42.349779Z","caller":"mvcc/index.go:214","msg":"compact tree index","revision":5616} {"level":"info","ts":"2026-04-13T03:02:42.455733Z","caller":"mvcc/kvstore_compaction.go:68","msg":"finished scheduled compaction","compact-revision":5616,"took":"104.785646ms","hash":1558206561,"current-db-size-bytes":54206464,"current-db-size":"54 MB","current-db-size-in-use-bytes":18235392,"current-db-size-in-use":"18 MB"} {"level":"info","ts":"2026-04-13T03:02:42.45579Z","caller":"mvcc/hash.go:137","msg":"storing new hash","hash":1558206561,"revision":5616,"compact-revision":2904}