{"level":"warn","ts":"2026-05-09T13:12:43.232574Z","caller":"embed/config.go:689","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-05-09T13:12:43.232687Z","caller":"etcdmain/etcd.go:73","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.204.45.109:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--experimental-initial-corrupt-check=true","--experimental-watch-progress-notify-interval=5s","--initial-advertise-peer-urls=https://199.204.45.109:2380","--initial-cluster=controller-1=https://199.204.45.229:2380,controller-2=https://199.204.45.109:2380","--initial-cluster-state=existing","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.204.45.109:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.204.45.109:2380","--name=controller-2","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt"]} {"level":"warn","ts":"2026-05-09T13:12:43.232832Z","caller":"embed/config.go:689","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-05-09T13:12:43.233085Z","caller":"embed/etcd.go:140","msg":"configuring peer listeners","listen-peer-urls":["https://199.204.45.109:2380"]} {"level":"info","ts":"2026-05-09T13:12:43.233117Z","caller":"embed/etcd.go:528","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-05-09T13:12:43.233785Z","caller":"embed/etcd.go:148","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.109:2379"]} {"level":"info","ts":"2026-05-09T13:12:43.233886Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.5.21","git-sha":"a17edfd","go-version":"go1.23.7","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-2","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.204.45.109:2380"],"listen-peer-urls":["https://199.204.45.109:2380"],"advertise-client-urls":["https://199.204.45.109:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.109:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://199.204.45.229:2380,controller-2=https://199.204.45.109:2380","initial-cluster-state":"existing","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"} {"level":"info","ts":"2026-05-09T13:12:43.240748Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"6.51413ms"} {"level":"info","ts":"2026-05-09T13:12:43.257710Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://199.204.45.229:2380","host":"199.204.45.229:2380","resolved-addr":"199.204.45.229:2380"} {"level":"info","ts":"2026-05-09T13:12:43.257784Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://199.204.45.109:2380","host":"199.204.45.109:2380","resolved-addr":"199.204.45.109:2380"} {"level":"info","ts":"2026-05-09T13:12:43.351304Z","caller":"etcdserver/raft.go:506","msg":"starting local member","local-member-id":"e2c0aeaa63762224","cluster-id":"4ad2674607ff1d60"} {"level":"info","ts":"2026-05-09T13:12:43.351439Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 switched to configuration voters=()"} {"level":"info","ts":"2026-05-09T13:12:43.351499Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became follower at term 0"} {"level":"info","ts":"2026-05-09T13:12:43.351515Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"newRaft e2c0aeaa63762224 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"warn","ts":"2026-05-09T13:12:43.355796Z","caller":"auth/store.go:1241","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-05-09T13:12:43.359851Z","caller":"mvcc/kvstore.go:425","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-05-09T13:12:43.360015Z","caller":"etcdserver/server.go:628","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-05-09T13:12:43.361710Z","caller":"etcdserver/quota.go:94","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-05-09T13:12:43.364127Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.364201Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db","remote-peer-urls":["https://199.204.45.229:2380"]} {"level":"info","ts":"2026-05-09T13:12:43.364244Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.364304Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.364568Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.364574Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.364852Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.364978Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.365046Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.365635Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db","remote-peer-urls":["https://199.204.45.229:2380"]} {"level":"info","ts":"2026-05-09T13:12:43.365823Z","caller":"etcdserver/server.go:875","msg":"starting etcd server","local-member-id":"e2c0aeaa63762224","local-server-version":"3.5.21","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-05-09T13:12:43.366504Z","caller":"etcdserver/server.go:775","msg":"starting initial election tick advance","election-ticks":10} {"level":"info","ts":"2026-05-09T13:12:43.366770Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-05-09T13:12:43.366818Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-05-09T13:12:43.366910Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-05-09T13:12:43.368290Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-05-09T13:12:43.371458Z","caller":"embed/etcd.go:762","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-05-09T13:12:43.371720Z","caller":"embed/etcd.go:633","msg":"serving peer traffic","address":"199.204.45.109:2380"} {"level":"info","ts":"2026-05-09T13:12:43.373538Z","caller":"embed/etcd.go:603","msg":"cmux::serve","address":"199.204.45.109:2380"} {"level":"info","ts":"2026-05-09T13:12:43.371839Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"e2c0aeaa63762224","initial-advertise-peer-urls":["https://199.204.45.109:2380"],"listen-peer-urls":["https://199.204.45.109:2380"],"advertise-client-urls":["https://199.204.45.109:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.109:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-05-09T13:12:43.371875Z","caller":"embed/etcd.go:908","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-05-09T13:12:43.374317Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [term: 0] received a MsgHeartbeat message with higher term from 3ba9ae53cc4335db [term: 2]"} {"level":"info","ts":"2026-05-09T13:12:43.374488Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became follower at term 2"} {"level":"info","ts":"2026-05-09T13:12:43.374623Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: e2c0aeaa63762224 elected leader 3ba9ae53cc4335db at term 2"} {"level":"info","ts":"2026-05-09T13:12:43.375428Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"e2c0aeaa63762224","to":"3ba9ae53cc4335db","stream-type":"stream Message"} {"level":"info","ts":"2026-05-09T13:12:43.375602Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.375854Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.375958Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.377339Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"e2c0aeaa63762224","to":"3ba9ae53cc4335db","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-05-09T13:12:43.377390Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.378574Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"e2c0aeaa63762224","remote-peer-id":"3ba9ae53cc4335db"} {"level":"info","ts":"2026-05-09T13:12:43.395995Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 switched to configuration voters=(4299158994234324443)"} {"level":"info","ts":"2026-05-09T13:12:43.396137Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"4ad2674607ff1d60","local-member-id":"e2c0aeaa63762224","added-peer-id":"3ba9ae53cc4335db","added-peer-peer-urls":["https://199.204.45.229:2380"],"added-peer-is-learner":false} {"level":"info","ts":"2026-05-09T13:12:43.396304Z","caller":"membership/cluster.go:587","msg":"set initial cluster version","cluster-id":"4ad2674607ff1d60","local-member-id":"e2c0aeaa63762224","cluster-version":"3.5"} {"level":"info","ts":"2026-05-09T13:12:43.396369Z","caller":"api/capability.go:75","msg":"enabled capabilities for version","cluster-version":"3.5"} {"level":"info","ts":"2026-05-09T13:12:43.433103Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 switched to configuration voters=(4299158994234324443) learners=(16339251494936519204)"} {"level":"info","ts":"2026-05-09T13:12:43.433457Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"4ad2674607ff1d60","local-member-id":"e2c0aeaa63762224","added-peer-id":"e2c0aeaa63762224","added-peer-peer-urls":["https://199.204.45.109:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-05-09T13:12:43.433909Z","caller":"etcdserver/server.go:2144","msg":"published local member to cluster through raft","local-member-id":"e2c0aeaa63762224","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.109:2379]}","request-path":"/0/members/e2c0aeaa63762224/attributes","cluster-id":"4ad2674607ff1d60","publish-timeout":"7s"} {"level":"info","ts":"2026-05-09T13:12:43.434028Z","caller":"embed/serve.go:124","msg":"ready to serve client requests"} {"level":"info","ts":"2026-05-09T13:12:43.434520Z","caller":"embed/serve.go:124","msg":"ready to serve client requests"} {"level":"info","ts":"2026-05-09T13:12:43.434608Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-05-09T13:12:43.434879Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-05-09T13:12:43.436665Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-05-09T13:12:43.437054Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-05-09T13:12:43.438638Z","caller":"embed/serve.go:275","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"info","ts":"2026-05-09T13:12:43.439650Z","caller":"embed/serve.go:275","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.204.45.109:2379"} {"level":"warn","ts":"2026-05-09T13:12:43.447682Z","caller":"embed/config_logging.go:170","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.109:37414","server-name":"","error":"EOF"} {"level":"info","ts":"2026-05-09T13:12:43.450641Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 switched to configuration voters=(4299158994234324443 16339251494936519204)"} {"level":"info","ts":"2026-05-09T13:12:43.450770Z","caller":"membership/cluster.go:537","msg":"promote member","cluster-id":"4ad2674607ff1d60","local-member-id":"e2c0aeaa63762224","promoted-member-id":"e2c0aeaa63762224"} {"level":"info","ts":"2026-05-09T13:12:45.181109Z","caller":"traceutil/trace.go:171","msg":"trace[1086628111] transaction","detail":"{read_only:false; response_revision:514; number_of_response:1; }","duration":"174.316725ms","start":"2026-05-09T13:12:45.006775Z","end":"2026-05-09T13:12:45.181091Z","steps":["trace[1086628111] 'process raft request' (duration: 174.271203ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:12:45.181163Z","caller":"traceutil/trace.go:171","msg":"trace[533642866] linearizableReadLoop","detail":"{readStateIndex:543; appliedIndex:544; }","duration":"212.666891ms","start":"2026-05-09T13:12:44.968467Z","end":"2026-05-09T13:12:45.181134Z","steps":["trace[533642866] 'read index received' (duration: 212.66036ms)","trace[533642866] 'applied index is now lower than readState.Index' (duration: 5.161µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:12:45.181419Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"212.901905ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.109\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:12:45.181466Z","caller":"traceutil/trace.go:171","msg":"trace[695887389] range","detail":"{range_begin:/registry/masterleases/199.204.45.109; range_end:; response_count:0; response_revision:514; }","duration":"212.993276ms","start":"2026-05-09T13:12:44.968461Z","end":"2026-05-09T13:12:45.181454Z","steps":["trace[695887389] 'agreement among raft nodes before linearized reading' (duration: 212.846974ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:12:45.182969Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"168.959756ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-05-09T13:12:45.183112Z","caller":"traceutil/trace.go:171","msg":"trace[912499538] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:514; }","duration":"169.134689ms","start":"2026-05-09T13:12:45.013960Z","end":"2026-05-09T13:12:45.183095Z","steps":["trace[912499538] 'agreement among raft nodes before linearized reading' (duration: 168.930585ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:12:46.489801Z","caller":"traceutil/trace.go:171","msg":"trace[1104537989] linearizableReadLoop","detail":"{readStateIndex:547; appliedIndex:547; }","duration":"148.70079ms","start":"2026-05-09T13:12:46.341063Z","end":"2026-05-09T13:12:46.489764Z","steps":["trace[1104537989] 'read index received' (duration: 148.69443ms)","trace[1104537989] 'applied index is now lower than readState.Index' (duration: 4.63µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:12:46.490053Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"148.952154ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rolebindings/kube-system/system::leader-locking-kube-scheduler\" limit:1 ","response":"range_response_count:1 size:808"} {"level":"info","ts":"2026-05-09T13:12:46.490160Z","caller":"traceutil/trace.go:171","msg":"trace[773117907] range","detail":"{range_begin:/registry/rolebindings/kube-system/system::leader-locking-kube-scheduler; range_end:; response_count:1; response_revision:517; }","duration":"149.102206ms","start":"2026-05-09T13:12:46.341035Z","end":"2026-05-09T13:12:46.490137Z","steps":["trace[773117907] 'agreement among raft nodes before linearized reading' (duration: 148.882062ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:12:47.794688Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"167.669928ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-05-09T13:12:47.794800Z","caller":"traceutil/trace.go:171","msg":"trace[1410612344] linearizableReadLoop","detail":"{readStateIndex:554; appliedIndex:553; }","duration":"193.452745ms","start":"2026-05-09T13:12:47.601330Z","end":"2026-05-09T13:12:47.794783Z","steps":["trace[1410612344] 'read index received' (duration: 25.771137ms)","trace[1410612344] 'applied index is now lower than readState.Index' (duration: 167.680528ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:12:47.794889Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"193.555917ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-05-09T13:12:47.794925Z","caller":"traceutil/trace.go:171","msg":"trace[1387827309] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:523; }","duration":"193.643809ms","start":"2026-05-09T13:12:47.601271Z","end":"2026-05-09T13:12:47.794914Z","steps":["trace[1387827309] 'agreement among raft nodes before linearized reading' (duration: 193.563697ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:12:55.738036Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559440991,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:12:55.781782Z","caller":"traceutil/trace.go:171","msg":"trace[1483953947] transaction","detail":"{read_only:false; response_revision:598; number_of_response:1; }","duration":"461.907486ms","start":"2026-05-09T13:12:55.319835Z","end":"2026-05-09T13:12:55.781742Z","steps":["trace[1483953947] 'process raft request' (duration: 461.711773ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:12:55.781836Z","caller":"traceutil/trace.go:171","msg":"trace[32680275] linearizableReadLoop","detail":"{readStateIndex:632; appliedIndex:631; }","duration":"544.452208ms","start":"2026-05-09T13:12:55.237349Z","end":"2026-05-09T13:12:55.781801Z","steps":["trace[32680275] 'read index received' (duration: 543.692824ms)","trace[32680275] 'applied index is now lower than readState.Index' (duration: 750.414µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:12:55.781970Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"544.605171ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-05-09T13:12:55.782078Z","caller":"traceutil/trace.go:171","msg":"trace[1231576489] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:598; }","duration":"544.776884ms","start":"2026-05-09T13:12:55.237286Z","end":"2026-05-09T13:12:55.782062Z","steps":["trace[1231576489] 'agreement among raft nodes before linearized reading' (duration: 544.603351ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:12:55.782123Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:12:55.237255Z","time spent":"544.857785ms","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:12:55.782218Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:12:55.319798Z","time spent":"462.060128ms","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-05-09T13:13:00.342072Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441011,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:13:00.842327Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441011,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:13:00.995427Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"1.203983632s","expected-duration":"1s"} {"level":"info","ts":"2026-05-09T13:13:01.288309Z","caller":"traceutil/trace.go:171","msg":"trace[1388370014] linearizableReadLoop","detail":"{readStateIndex:644; appliedIndex:644; }","duration":"1.446817183s","start":"2026-05-09T13:12:59.841456Z","end":"2026-05-09T13:13:01.288273Z","steps":["trace[1388370014] 'read index received' (duration: 1.446804043s)","trace[1388370014] 'applied index is now lower than readState.Index' (duration: 10.57µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:01.325172Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.483673593s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-05-09T13:13:01.325260Z","caller":"traceutil/trace.go:171","msg":"trace[828294605] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:608; }","duration":"1.483845246s","start":"2026-05-09T13:12:59.841394Z","end":"2026-05-09T13:13:01.325240Z","steps":["trace[828294605] 'agreement among raft nodes before linearized reading' (duration: 1.446976105s)","trace[828294605] 'range keys from in-memory index tree' (duration: 36.687217ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:01.325303Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:12:59.841367Z","time spent":"1.483926259s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:13:01.657318Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"379.48782ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-05-09T13:13:01.657994Z","caller":"traceutil/trace.go:171","msg":"trace[2070584503] transaction","detail":"{read_only:false; response_revision:610; number_of_response:1; }","duration":"1.737820571s","start":"2026-05-09T13:12:59.920148Z","end":"2026-05-09T13:13:01.657968Z","steps":["trace[2070584503] 'process raft request' (duration: 1.737496475s)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:13:01.658450Z","caller":"traceutil/trace.go:171","msg":"trace[1845466237] transaction","detail":"{read_only:false; response_revision:611; number_of_response:1; }","duration":"635.292427ms","start":"2026-05-09T13:13:01.023087Z","end":"2026-05-09T13:13:01.658379Z","steps":["trace[1845466237] 'process raft request' (duration: 634.83055ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:13:01.658551Z","caller":"traceutil/trace.go:171","msg":"trace[1878608160] linearizableReadLoop","detail":"{readStateIndex:649; appliedIndex:644; }","duration":"370.120423ms","start":"2026-05-09T13:13:01.288410Z","end":"2026-05-09T13:13:01.658530Z","steps":["trace[1878608160] 'read index received' (duration: 1.902562ms)","trace[1878608160] 'applied index is now lower than readState.Index' (duration: 368.216641ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:01.658696Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:12:59.920120Z","time spent":"1.737934373s","remote":"127.0.0.1:34570","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3889,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-05-09T13:13:01.658730Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:01.023036Z","time spent":"635.560382ms","remote":"127.0.0.1:34490","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":761,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-05-09T13:13:01.658805Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.746752603s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"warn","ts":"2026-05-09T13:13:01.658932Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.365490314s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:01.658854Z","caller":"traceutil/trace.go:171","msg":"trace[501299296] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:613; }","duration":"1.746892196s","start":"2026-05-09T13:12:59.911944Z","end":"2026-05-09T13:13:01.658836Z","steps":["trace[501299296] 'agreement among raft nodes before linearized reading' (duration: 1.746656841s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.659078Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"636.037831ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/kube-system/kube-proxy\" limit:1 ","response":"range_response_count:1 size:185"} {"level":"info","ts":"2026-05-09T13:13:01.659109Z","caller":"traceutil/trace.go:171","msg":"trace[1030915103] range","detail":"{range_begin:/registry/serviceaccounts/kube-system/kube-proxy; range_end:; response_count:1; response_revision:613; }","duration":"636.125652ms","start":"2026-05-09T13:13:01.022975Z","end":"2026-05-09T13:13:01.659101Z","steps":["trace[1030915103] 'agreement among raft nodes before linearized reading' (duration: 635.671305ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.659116Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.153611278s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-05-09T13:13:01.659157Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:12:59.911908Z","time spent":"1.74720383s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"info","ts":"2026-05-09T13:13:01.659227Z","caller":"traceutil/trace.go:171","msg":"trace[776311195] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:613; }","duration":"1.153788081s","start":"2026-05-09T13:13:00.505421Z","end":"2026-05-09T13:13:01.659209Z","steps":["trace[776311195] 'agreement among raft nodes before linearized reading' (duration: 1.15317187s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.659470Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.659788614s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-05-09T13:13:01.658946Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"432.26643ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-proxy-9tfmb\" limit:1 ","response":"range_response_count:1 size:3429"} {"level":"info","ts":"2026-05-09T13:13:01.659502Z","caller":"traceutil/trace.go:171","msg":"trace[642032995] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:613; }","duration":"1.659871706s","start":"2026-05-09T13:12:59.999621Z","end":"2026-05-09T13:13:01.659493Z","steps":["trace[642032995] 'agreement among raft nodes before linearized reading' (duration: 1.659037112s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.659543Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:12:59.999579Z","time spent":"1.659957157s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:13:01.659505Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:00.505386Z","time spent":"1.154106217s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-05-09T13:13:01.659547Z","caller":"traceutil/trace.go:171","msg":"trace[1408940406] range","detail":"{range_begin:/registry/pods/kube-system/kube-proxy-9tfmb; range_end:; response_count:1; response_revision:613; }","duration":"432.925891ms","start":"2026-05-09T13:13:01.226583Z","end":"2026-05-09T13:13:01.659508Z","steps":["trace[1408940406] 'agreement among raft nodes before linearized reading' (duration: 432.049755ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.659174Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:01.022937Z","time spent":"636.225793ms","remote":"127.0.0.1:34596","response type":"/etcdserverpb.KV/Range","request count":0,"request size":52,"response count":1,"response size":208,"request content":"key:\"/registry/serviceaccounts/kube-system/kube-proxy\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:13:01.659612Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:01.226558Z","time spent":"433.040712ms","remote":"127.0.0.1:34580","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":3452,"request content":"key:\"/registry/pods/kube-system/kube-proxy-9tfmb\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:13:01.658844Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.742269867s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/limitranges\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:01.659848Z","caller":"traceutil/trace.go:171","msg":"trace[642374155] range","detail":"{range_begin:/registry/limitranges; range_end:; response_count:0; response_revision:613; }","duration":"1.743349364s","start":"2026-05-09T13:12:59.916462Z","end":"2026-05-09T13:13:01.659812Z","steps":["trace[642374155] 'agreement among raft nodes before linearized reading' (duration: 1.742248616s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.659933Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:12:59.916443Z","time spent":"1.743462666s","remote":"127.0.0.1:34540","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/limitranges\" limit:1 "} {"level":"info","ts":"2026-05-09T13:13:01.659010Z","caller":"traceutil/trace.go:171","msg":"trace[594743623] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:613; }","duration":"1.365558145s","start":"2026-05-09T13:13:00.293427Z","end":"2026-05-09T13:13:01.658985Z","steps":["trace[594743623] 'agreement among raft nodes before linearized reading' (duration: 1.36524377s)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:13:01.967524Z","caller":"traceutil/trace.go:171","msg":"trace[382160855] transaction","detail":"{read_only:false; response_revision:614; number_of_response:1; }","duration":"297.467859ms","start":"2026-05-09T13:13:01.670023Z","end":"2026-05-09T13:13:01.967491Z","steps":["trace[382160855] 'process raft request' (duration: 297.300056ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:13:01.974127Z","caller":"traceutil/trace.go:171","msg":"trace[981295071] transaction","detail":"{read_only:false; number_of_response:0; response_revision:614; }","duration":"304.058859ms","start":"2026-05-09T13:13:01.670052Z","end":"2026-05-09T13:13:01.974111Z","steps":["trace[981295071] 'process raft request' (duration: 303.856576ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.974222Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:01.670033Z","time spent":"304.145591ms","remote":"127.0.0.1:34580","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":28,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-05-09T13:13:01.983016Z","caller":"traceutil/trace.go:171","msg":"trace[251716908] transaction","detail":"{read_only:false; response_revision:615; number_of_response:1; }","duration":"299.813669ms","start":"2026-05-09T13:13:01.683178Z","end":"2026-05-09T13:13:01.982992Z","steps":["trace[251716908] 'process raft request' (duration: 299.574445ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.983874Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:01.683162Z","time spent":"300.057562ms","remote":"127.0.0.1:34580","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4839,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:13:01.984537Z","caller":"traceutil/trace.go:171","msg":"trace[1019703994] linearizableReadLoop","detail":"{readStateIndex:651; appliedIndex:654; }","duration":"311.233901ms","start":"2026-05-09T13:13:01.673272Z","end":"2026-05-09T13:13:01.984505Z","steps":["trace[1019703994] 'read index received' (duration: 311.224151ms)","trace[1019703994] 'applied index is now lower than readState.Index' (duration: 8.05µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:01.984931Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"311.564786ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:01.985052Z","caller":"traceutil/trace.go:171","msg":"trace[1664571342] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:615; }","duration":"311.885312ms","start":"2026-05-09T13:13:01.673149Z","end":"2026-05-09T13:13:01.985034Z","steps":["trace[1664571342] 'agreement among raft nodes before linearized reading' (duration: 311.495225ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:01.985091Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:01.673092Z","time spent":"311.989043ms","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-05-09T13:13:08.451217Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"1.428413366s","expected-duration":"1s"} {"level":"warn","ts":"2026-05-09T13:13:08.530249Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.224487591s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:08.530336Z","caller":"traceutil/trace.go:171","msg":"trace[884841501] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:643; }","duration":"1.224593263s","start":"2026-05-09T13:13:07.305723Z","end":"2026-05-09T13:13:08.530317Z","steps":["trace[884841501] 'range keys from in-memory index tree' (duration: 1.224426701s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:08.530388Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.698182901s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-05-09T13:13:08.530447Z","caller":"traceutil/trace.go:171","msg":"trace[1146630085] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:643; }","duration":"1.698287392s","start":"2026-05-09T13:13:06.832141Z","end":"2026-05-09T13:13:08.530428Z","steps":["trace[1146630085] 'agreement among raft nodes before linearized reading' (duration: 26.917222ms)","trace[1146630085] 'range keys from in-memory index tree' (duration: 1.671261838s)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:08.530508Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:06.832116Z","time spent":"1.698373094s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:13:08.530443Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.783770159s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:2 size:263"} {"level":"info","ts":"2026-05-09T13:13:08.530610Z","caller":"traceutil/trace.go:171","msg":"trace[125743344] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:2; response_revision:643; }","duration":"1.783988552s","start":"2026-05-09T13:13:06.746595Z","end":"2026-05-09T13:13:08.530583Z","steps":["trace[125743344] 'range keys from in-memory index tree' (duration: 1.782187572s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:08.530723Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:06.746583Z","time spent":"1.784104044s","remote":"127.0.0.1:34440","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":2,"response size":286,"request content":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" "} {"level":"info","ts":"2026-05-09T13:13:08.757917Z","caller":"traceutil/trace.go:171","msg":"trace[371701542] linearizableReadLoop","detail":"{readStateIndex:686; appliedIndex:686; }","duration":"252.09935ms","start":"2026-05-09T13:13:08.505794Z","end":"2026-05-09T13:13:08.757894Z","steps":["trace[371701542] 'read index received' (duration: 252.08992ms)","trace[371701542] 'applied index is now lower than readState.Index' (duration: 8.22µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:08.758046Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"252.239732ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:08.758144Z","caller":"traceutil/trace.go:171","msg":"trace[1366602487] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:645; }","duration":"252.393274ms","start":"2026-05-09T13:13:08.505730Z","end":"2026-05-09T13:13:08.758123Z","steps":["trace[1366602487] 'agreement among raft nodes before linearized reading' (duration: 252.253052ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:08.759589Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"251.152843ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"warn","ts":"2026-05-09T13:13:08.759581Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"229.188314ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:08.759633Z","caller":"traceutil/trace.go:171","msg":"trace[2071515530] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:645; }","duration":"251.257685ms","start":"2026-05-09T13:13:08.508366Z","end":"2026-05-09T13:13:08.759624Z","steps":["trace[2071515530] 'agreement among raft nodes before linearized reading' (duration: 251.172144ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:13:08.759657Z","caller":"traceutil/trace.go:171","msg":"trace[108158904] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:645; }","duration":"229.268156ms","start":"2026-05-09T13:13:08.530376Z","end":"2026-05-09T13:13:08.759644Z","steps":["trace[108158904] 'agreement among raft nodes before linearized reading' (duration: 229.169764ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:08.759744Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"226.654875ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:498"} {"level":"info","ts":"2026-05-09T13:13:08.759852Z","caller":"traceutil/trace.go:171","msg":"trace[1608790117] range","detail":"{range_begin:/registry/endpointslices/default/kubernetes; range_end:; response_count:1; response_revision:645; }","duration":"226.823028ms","start":"2026-05-09T13:13:08.533006Z","end":"2026-05-09T13:13:08.759829Z","steps":["trace[1608790117] 'agreement among raft nodes before linearized reading' (duration: 226.601214ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:12.090715Z","caller":"embed/config_logging.go:170","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.77:41668","server-name":"","error":"EOF"} {"level":"info","ts":"2026-05-09T13:13:12.536753Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 switched to configuration voters=(4299158994234324443 16339251494936519204) learners=(18083212291503342128)"} {"level":"info","ts":"2026-05-09T13:13:12.536974Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"4ad2674607ff1d60","local-member-id":"e2c0aeaa63762224","added-peer-id":"faf479c52908c230","added-peer-peer-urls":["https://199.204.45.77:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-05-09T13:13:12.537009Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:12.537091Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:12.537904Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:12.538476Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:12.543585Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:12.543635Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:12.543672Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230","remote-peer-urls":["https://199.204.45.77:2380"]} {"level":"info","ts":"2026-05-09T13:13:12.543603Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:12.543778Z","caller":"etcdserver/server.go:2021","msg":"applied a configuration change through raft","local-member-id":"e2c0aeaa63762224","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"faf479c52908c230"} {"level":"warn","ts":"2026-05-09T13:13:12.596281Z","caller":"embed/config_logging.go:170","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.77:41746","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-05-09T13:13:14.097737Z","caller":"embed/config_logging.go:170","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.77:41770","server-name":"","error":"EOF"} {"level":"info","ts":"2026-05-09T13:13:15.082299Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:15.082391Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:15.082836Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:15.089572Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"e2c0aeaa63762224","to":"faf479c52908c230","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-05-09T13:13:15.089619Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:15.094810Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"e2c0aeaa63762224","to":"faf479c52908c230","stream-type":"stream Message"} {"level":"info","ts":"2026-05-09T13:13:15.094852Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"e2c0aeaa63762224","remote-peer-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:15.634396Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 switched to configuration voters=(4299158994234324443 16339251494936519204 18083212291503342128)"} {"level":"info","ts":"2026-05-09T13:13:15.634527Z","caller":"membership/cluster.go:537","msg":"promote member","cluster-id":"4ad2674607ff1d60","local-member-id":"e2c0aeaa63762224","promoted-member-id":"faf479c52908c230"} {"level":"info","ts":"2026-05-09T13:13:26.734275Z","caller":"traceutil/trace.go:171","msg":"trace[1063962781] linearizableReadLoop","detail":"{readStateIndex:794; appliedIndex:794; }","duration":"146.668383ms","start":"2026-05-09T13:13:26.587581Z","end":"2026-05-09T13:13:26.734249Z","steps":["trace[1063962781] 'read index received' (duration: 146.660783ms)","trace[1063962781] 'applied index is now lower than readState.Index' (duration: 6.23µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:26.734453Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"146.851817ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.109\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-05-09T13:13:26.734546Z","caller":"traceutil/trace.go:171","msg":"trace[1839366131] range","detail":"{range_begin:/registry/masterleases/199.204.45.109; range_end:; response_count:1; response_revision:741; }","duration":"147.021599ms","start":"2026-05-09T13:13:26.587502Z","end":"2026-05-09T13:13:26.734524Z","steps":["trace[1839366131] 'agreement among raft nodes before linearized reading' (duration: 146.877277ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:13:26.924599Z","caller":"traceutil/trace.go:171","msg":"trace[1890893297] linearizableReadLoop","detail":"{readStateIndex:796; appliedIndex:796; }","duration":"121.281508ms","start":"2026-05-09T13:13:26.803291Z","end":"2026-05-09T13:13:26.924572Z","steps":["trace[1890893297] 'read index received' (duration: 121.274558ms)","trace[1890893297] 'applied index is now lower than readState.Index' (duration: 5.61µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:27.006538Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"203.22347ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-05-09T13:13:27.006632Z","caller":"traceutil/trace.go:171","msg":"trace[931859784] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:742; }","duration":"203.373052ms","start":"2026-05-09T13:13:26.803236Z","end":"2026-05-09T13:13:27.006609Z","steps":["trace[931859784] 'agreement among raft nodes before linearized reading' (duration: 121.484232ms)","trace[931859784] 'range keys from in-memory index tree' (duration: 81.726318ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:27.006799Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"177.277616ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:461"} {"level":"info","ts":"2026-05-09T13:13:27.006905Z","caller":"traceutil/trace.go:171","msg":"trace[341381938] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:743; }","duration":"177.449119ms","start":"2026-05-09T13:13:26.829437Z","end":"2026-05-09T13:13:27.006886Z","steps":["trace[341381938] 'agreement among raft nodes before linearized reading' (duration: 177.243186ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:13:34.838293Z","caller":"traceutil/trace.go:171","msg":"trace[1219606769] linearizableReadLoop","detail":"{readStateIndex:859; appliedIndex:860; }","duration":"302.55644ms","start":"2026-05-09T13:13:34.535715Z","end":"2026-05-09T13:13:34.838271Z","steps":["trace[1219606769] 'read index received' (duration: 302.55125ms)","trace[1219606769] 'applied index is now lower than readState.Index' (duration: 4.51µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:34.838407Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"302.691522ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:34.838434Z","caller":"traceutil/trace.go:171","msg":"trace[401578410] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:797; }","duration":"302.779643ms","start":"2026-05-09T13:13:34.535646Z","end":"2026-05-09T13:13:34.838426Z","steps":["trace[401578410] 'agreement among raft nodes before linearized reading' (duration: 302.717392ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:34.838459Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:34.535619Z","time spent":"302.834764ms","remote":"127.0.0.1:34420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-05-09T13:13:39.639955Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"123.031765ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-05-09T13:13:45.243659Z","caller":"traceutil/trace.go:171","msg":"trace[713282614] linearizableReadLoop","detail":"{readStateIndex:899; appliedIndex:899; }","duration":"334.41748ms","start":"2026-05-09T13:13:44.909214Z","end":"2026-05-09T13:13:45.243631Z","steps":["trace[713282614] 'read index received' (duration: 334.41287ms)","trace[713282614] 'applied index is now lower than readState.Index' (duration: 3.57µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:45.243861Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"334.623633ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/podtemplates/\" range_end:\"/registry/podtemplates0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:45.243950Z","caller":"traceutil/trace.go:171","msg":"trace[1600095048] range","detail":"{range_begin:/registry/podtemplates/; range_end:/registry/podtemplates0; response_count:0; response_revision:830; }","duration":"334.735164ms","start":"2026-05-09T13:13:44.909195Z","end":"2026-05-09T13:13:45.243930Z","steps":["trace[1600095048] 'agreement among raft nodes before linearized reading' (duration: 334.573812ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:13:45.243994Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:13:44.909119Z","time spent":"334.867986ms","remote":"127.0.0.1:34532","response type":"/etcdserverpb.KV/Range","request count":0,"request size":52,"response count":0,"response size":28,"request content":"key:\"/registry/podtemplates/\" range_end:\"/registry/podtemplates0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:13:45.245621Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"272.898212ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourcequotas/\" range_end:\"/registry/resourcequotas0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:45.245662Z","caller":"traceutil/trace.go:171","msg":"trace[1766088729] range","detail":"{range_begin:/registry/resourcequotas/; range_end:/registry/resourcequotas0; response_count:0; response_revision:830; }","duration":"272.992133ms","start":"2026-05-09T13:13:44.972661Z","end":"2026-05-09T13:13:45.245653Z","steps":["trace[1766088729] 'agreement among raft nodes before linearized reading' (duration: 272.917272ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:13:45.454772Z","caller":"traceutil/trace.go:171","msg":"trace[316465820] linearizableReadLoop","detail":"{readStateIndex:900; appliedIndex:900; }","duration":"161.573902ms","start":"2026-05-09T13:13:45.293183Z","end":"2026-05-09T13:13:45.454757Z","steps":["trace[316465820] 'read index received' (duration: 161.570492ms)","trace[316465820] 'applied index is now lower than readState.Index' (duration: 2.77µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:13:45.454881Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"161.681994ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:13:45.454937Z","caller":"traceutil/trace.go:171","msg":"trace[77607393] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:831; }","duration":"161.752335ms","start":"2026-05-09T13:13:45.293173Z","end":"2026-05-09T13:13:45.454925Z","steps":["trace[77607393] 'agreement among raft nodes before linearized reading' (duration: 161.662094ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:03.793754Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:04.294767Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:04.795740Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:05.296688Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:05.503692Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000073857s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:05.503793Z","caller":"traceutil/trace.go:171","msg":"trace[1963096781] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000234429s","start":"2026-05-09T13:14:03.503538Z","end":"2026-05-09T13:14:05.503772Z","steps":["trace[1963096781] 'agreement among raft nodes before linearized reading' (duration: 2.000115348s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:05.503857Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:03.503519Z","time spent":"2.00032878s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/05/09 13:14:05 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-05-09T13:14:05.797061Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:06.298146Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:06.799052Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:07.047803Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"3.961628527s","expected-duration":"1s"} {"level":"warn","ts":"2026-05-09T13:14:07.299267Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:07.503929Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.999845846s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:07.504044Z","caller":"traceutil/trace.go:171","msg":"trace[365671656] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000072438s","start":"2026-05-09T13:14:05.503947Z","end":"2026-05-09T13:14:07.504019Z","steps":["trace[365671656] 'agreement among raft nodes before linearized reading' (duration: 1.999926486s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:07.504104Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:05.503923Z","time spent":"2.00016651s","remote":"127.0.0.1:34420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/05/09 13:14:07 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-05-09T13:14:07.783936Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000887603s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-05-09T13:14:07.784099Z","caller":"traceutil/trace.go:171","msg":"trace[1704214545] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.001105996s","start":"2026-05-09T13:14:05.782945Z","end":"2026-05-09T13:14:07.784051Z","steps":["trace[1704214545] 'agreement among raft nodes before linearized reading' (duration: 2.000911714s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:07.784231Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:05.782932Z","time spent":"2.001231778s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/05/09 13:14:07 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-05-09T13:14:07.799897Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:08.300530Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441334,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:08.453579Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 is starting a new election at term 2"} {"level":"info","ts":"2026-05-09T13:14:08.453682Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became pre-candidate at term 2"} {"level":"info","ts":"2026-05-09T13:14:08.453704Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 received MsgPreVoteResp from e2c0aeaa63762224 at term 2"} {"level":"info","ts":"2026-05-09T13:14:08.453732Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1054] sent MsgPreVote request to 3ba9ae53cc4335db at term 2"} {"level":"info","ts":"2026-05-09T13:14:08.453746Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1054] sent MsgPreVote request to faf479c52908c230 at term 2"} {"level":"info","ts":"2026-05-09T13:14:08.453763Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: e2c0aeaa63762224 lost leader 3ba9ae53cc4335db at term 2"} {"level":"info","ts":"2026-05-09T13:14:08.705733Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became follower at term 2"} {"level":"info","ts":"2026-05-09T13:14:08.705802Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: e2c0aeaa63762224 elected leader 3ba9ae53cc4335db at term 2"} {"level":"warn","ts":"2026-05-09T13:14:08.706085Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.413062377s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-05-09T13:14:08.706205Z","caller":"traceutil/trace.go:171","msg":"trace[544306998] range","detail":"{range_begin:; range_end:; }","duration":"5.413209689s","start":"2026-05-09T13:14:03.292971Z","end":"2026-05-09T13:14:08.706180Z","steps":["trace[544306998] 'agreement among raft nodes before linearized reading' (duration: 5.413058246s)"],"step_count":1} {"level":"error","ts":"2026-05-09T13:14:08.706274Z","caller":"etcdhttp/health.go:370","msg":"Health check error","path":"/readyz","reason":"[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: leader changed\n[+]non_learner ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHttpEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:370\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-05-09T13:14:08.767939Z","caller":"etcdserver/v3_server.go:897","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":2460265074559441338,"received-request-id":2460265074559441334} {"level":"info","ts":"2026-05-09T13:14:08.768336Z","caller":"traceutil/trace.go:171","msg":"trace[1242109914] transaction","detail":"{read_only:false; response_revision:975; number_of_response:1; }","duration":"1.20876852s","start":"2026-05-09T13:14:07.559539Z","end":"2026-05-09T13:14:08.768308Z","steps":["trace[1242109914] 'process raft request' (duration: 1.208720429s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.768471Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:07.559504Z","time spent":"1.208881262s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:14:08.768556Z","caller":"traceutil/trace.go:171","msg":"trace[15263722] transaction","detail":"{read_only:false; response_revision:973; number_of_response:1; }","duration":"2.988138164s","start":"2026-05-09T13:14:05.780392Z","end":"2026-05-09T13:14:08.768530Z","steps":["trace[15263722] 'process raft request' (duration: 2.987686778s)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:08.768738Z","caller":"traceutil/trace.go:171","msg":"trace[1110190126] transaction","detail":"{read_only:false; response_revision:974; number_of_response:1; }","duration":"1.261035005s","start":"2026-05-09T13:14:07.507690Z","end":"2026-05-09T13:14:08.768725Z","steps":["trace[1110190126] 'process raft request' (duration: 1.260527808s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.768808Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:07.507661Z","time spent":"1.261115336s","remote":"127.0.0.1:34490","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-05-09T13:14:08.768961Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:05.780350Z","time spent":"2.988323937s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-05-09T13:14:08.789406Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.408992543s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"warn","ts":"2026-05-09T13:14:08.789436Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.483192547s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-05-09T13:14:08.789500Z","caller":"traceutil/trace.go:171","msg":"trace[677961863] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:977; }","duration":"2.409140235s","start":"2026-05-09T13:14:06.380340Z","end":"2026-05-09T13:14:08.789480Z","steps":["trace[677961863] 'agreement among raft nodes before linearized reading' (duration: 2.408972883s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.789551Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:06.380320Z","time spent":"2.409221187s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":506,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"info","ts":"2026-05-09T13:14:08.789543Z","caller":"traceutil/trace.go:171","msg":"trace[729794796] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:977; }","duration":"2.483356648s","start":"2026-05-09T13:14:06.306168Z","end":"2026-05-09T13:14:08.789525Z","steps":["trace[729794796] 'agreement among raft nodes before linearized reading' (duration: 2.483169826s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.789598Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:06.306141Z","time spent":"2.48345114s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:08.789754Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.554216997s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/\" range_end:\"/registry/leases0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-05-09T13:14:08.789779Z","caller":"traceutil/trace.go:171","msg":"trace[667442345] range","detail":"{range_begin:/registry/leases/; range_end:/registry/leases0; response_count:0; response_revision:977; }","duration":"2.554280768s","start":"2026-05-09T13:14:06.235493Z","end":"2026-05-09T13:14:08.789773Z","steps":["trace[667442345] 'agreement among raft nodes before linearized reading' (duration: 2.554238747s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.789793Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:06.235474Z","time spent":"2.554315129s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":10,"response size":30,"request content":"key:\"/registry/leases/\" range_end:\"/registry/leases0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:08.789824Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.276212328s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:08.789858Z","caller":"traceutil/trace.go:171","msg":"trace[1862425397] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:977; }","duration":"1.27626638s","start":"2026-05-09T13:14:07.513578Z","end":"2026-05-09T13:14:08.789845Z","steps":["trace[1862425397] 'agreement among raft nodes before linearized reading' (duration: 1.276214569s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.789891Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:07.513561Z","time spent":"1.27632514s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-05-09T13:14:08.789922Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.348944692s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-05-09T13:14:08.789938Z","caller":"traceutil/trace.go:171","msg":"trace[1580737690] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:977; }","duration":"4.348985602s","start":"2026-05-09T13:14:04.440947Z","end":"2026-05-09T13:14:08.789933Z","steps":["trace[1580737690] 'agreement among raft nodes before linearized reading' (duration: 4.348959601s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.789952Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:04.440928Z","time spent":"4.349020763s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:08.790065Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.653379297s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-05-09T13:14:08.790107Z","caller":"traceutil/trace.go:171","msg":"trace[1173973836] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:977; }","duration":"4.653451538s","start":"2026-05-09T13:14:04.136650Z","end":"2026-05-09T13:14:08.790102Z","steps":["trace[1173973836] 'agreement among raft nodes before linearized reading' (duration: 4.653393197s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.790127Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:04.136630Z","time spent":"4.653492258s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:08.789898Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.574696494s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csistoragecapacities/\" range_end:\"/registry/csistoragecapacities0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:08.790300Z","caller":"traceutil/trace.go:171","msg":"trace[2123693775] range","detail":"{range_begin:/registry/csistoragecapacities/; range_end:/registry/csistoragecapacities0; response_count:0; response_revision:977; }","duration":"2.575099961s","start":"2026-05-09T13:14:06.215158Z","end":"2026-05-09T13:14:08.790258Z","steps":["trace[2123693775] 'agreement among raft nodes before linearized reading' (duration: 2.574720915s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.790306Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.102300285s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/deployments/\" range_end:\"/registry/deployments0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"warn","ts":"2026-05-09T13:14:08.790400Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:06.215108Z","time spent":"2.575251203s","remote":"127.0.0.1:34832","response type":"/etcdserverpb.KV/Range","request count":0,"request size":68,"response count":0,"response size":28,"request content":"key:\"/registry/csistoragecapacities/\" range_end:\"/registry/csistoragecapacities0\" count_only:true "} {"level":"info","ts":"2026-05-09T13:14:08.790402Z","caller":"traceutil/trace.go:171","msg":"trace[238988479] range","detail":"{range_begin:/registry/deployments/; range_end:/registry/deployments0; response_count:0; response_revision:977; }","duration":"5.102462697s","start":"2026-05-09T13:14:03.687919Z","end":"2026-05-09T13:14:08.790381Z","steps":["trace[238988479] 'agreement among raft nodes before linearized reading' (duration: 5.102284303s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.790396Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.315079824s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumeclaims/\" range_end:\"/registry/persistentvolumeclaims0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-05-09T13:14:08.790407Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.200561955s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.109\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"warn","ts":"2026-05-09T13:14:08.790449Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:03.687895Z","time spent":"5.102547418s","remote":"127.0.0.1:34868","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":2,"response size":30,"request content":"key:\"/registry/deployments/\" range_end:\"/registry/deployments0\" count_only:true "} {"level":"info","ts":"2026-05-09T13:14:08.790460Z","caller":"traceutil/trace.go:171","msg":"trace[687143132] range","detail":"{range_begin:/registry/persistentvolumeclaims/; range_end:/registry/persistentvolumeclaims0; response_count:0; response_revision:977; }","duration":"2.315190725s","start":"2026-05-09T13:14:06.475254Z","end":"2026-05-09T13:14:08.790444Z","steps":["trace[687143132] 'agreement among raft nodes before linearized reading' (duration: 2.315070174s)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:08.790471Z","caller":"traceutil/trace.go:171","msg":"trace[1571082143] range","detail":"{range_begin:/registry/masterleases/199.204.45.109; range_end:; response_count:1; response_revision:977; }","duration":"2.200683727s","start":"2026-05-09T13:14:06.589772Z","end":"2026-05-09T13:14:08.790456Z","steps":["trace[1571082143] 'agreement among raft nodes before linearized reading' (duration: 2.200549065s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.790497Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:06.475228Z","time spent":"2.315259596s","remote":"127.0.0.1:34550","response type":"/etcdserverpb.KV/Range","request count":0,"request size":72,"response count":0,"response size":28,"request content":"key:\"/registry/persistentvolumeclaims/\" range_end:\"/registry/persistentvolumeclaims0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:08.790508Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:06.589743Z","time spent":"2.200759857s","remote":"127.0.0.1:34440","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.109\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:08.882234Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"173.852211ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:08.883332Z","caller":"traceutil/trace.go:171","msg":"trace[684661509] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:995; }","duration":"174.978517ms","start":"2026-05-09T13:14:08.708330Z","end":"2026-05-09T13:14:08.883308Z","steps":["trace[684661509] 'agreement among raft nodes before linearized reading' (duration: 173.80263ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.969374Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"126.450032ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1alpha1.gateway.networking.x-k8s.io\" limit:1 ","response":"range_response_count:1 size:1019"} {"level":"info","ts":"2026-05-09T13:14:08.969451Z","caller":"traceutil/trace.go:171","msg":"trace[1526620326] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1alpha1.gateway.networking.x-k8s.io; range_end:; response_count:1; response_revision:1006; }","duration":"126.559834ms","start":"2026-05-09T13:14:08.842873Z","end":"2026-05-09T13:14:08.969433Z","steps":["trace[1526620326] 'agreement among raft nodes before linearized reading' (duration: 119.925125ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.969495Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"126.623105ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1alpha2.gateway.networking.k8s.io\" limit:1 ","response":"range_response_count:1 size:1013"} {"level":"info","ts":"2026-05-09T13:14:08.969590Z","caller":"traceutil/trace.go:171","msg":"trace[1442524021] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1alpha2.gateway.networking.k8s.io; range_end:; response_count:1; response_revision:1006; }","duration":"126.776938ms","start":"2026-05-09T13:14:08.842790Z","end":"2026-05-09T13:14:08.969567Z","steps":["trace[1442524021] 'agreement among raft nodes before linearized reading' (duration: 120.625495ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.969596Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"176.582488ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1beta1.gateway.networking.k8s.io\" limit:1 ","response":"range_response_count:1 size:1010"} {"level":"info","ts":"2026-05-09T13:14:08.969661Z","caller":"traceutil/trace.go:171","msg":"trace[1844681060] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1beta1.gateway.networking.k8s.io; range_end:; response_count:1; response_revision:1006; }","duration":"176.68797ms","start":"2026-05-09T13:14:08.792957Z","end":"2026-05-09T13:14:08.969645Z","steps":["trace[1844681060] 'agreement among raft nodes before linearized reading' (duration: 170.493616ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:08.969443Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"177.955286ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-05-09T13:14:08.969919Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"126.558435ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1.gateway.networking.k8s.io\" limit:1 ","response":"range_response_count:1 size:995"} {"level":"info","ts":"2026-05-09T13:14:08.969984Z","caller":"traceutil/trace.go:171","msg":"trace[2033966400] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1.gateway.networking.k8s.io; range_end:; response_count:1; response_revision:1006; }","duration":"126.762428ms","start":"2026-05-09T13:14:08.843209Z","end":"2026-05-09T13:14:08.969972Z","steps":["trace[2033966400] 'agreement among raft nodes before linearized reading' (duration: 119.5836ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:08.969882Z","caller":"traceutil/trace.go:171","msg":"trace[1323100095] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1006; }","duration":"178.434003ms","start":"2026-05-09T13:14:08.791420Z","end":"2026-05-09T13:14:08.969854Z","steps":["trace[1323100095] 'agreement among raft nodes before linearized reading' (duration: 171.357077ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:10.403002Z","caller":"traceutil/trace.go:171","msg":"trace[1409899630] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1035; }","duration":"935.975536ms","start":"2026-05-09T13:14:09.467013Z","end":"2026-05-09T13:14:10.402989Z","steps":["trace[1409899630] 'process raft request' (duration: 935.910575ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:10.403282Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:09.466975Z","time spent":"936.146028ms","remote":"127.0.0.1:34472","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":105,"response count":0,"response size":98740,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:14:10.403788Z","caller":"traceutil/trace.go:171","msg":"trace[1895782805] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1035; }","duration":"954.90519ms","start":"2026-05-09T13:14:09.448151Z","end":"2026-05-09T13:14:10.403056Z","steps":["trace[1895782805] 'process raft request' (duration: 954.689597ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:10.405631Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:09.448113Z","time spent":"957.304433ms","remote":"127.0.0.1:34472","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":97,"response count":0,"response size":151272,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:14:10.920160Z","caller":"traceutil/trace.go:171","msg":"trace[1342182906] linearizableReadLoop","detail":"{readStateIndex:1194; appliedIndex:1196; }","duration":"416.282183ms","start":"2026-05-09T13:14:10.503854Z","end":"2026-05-09T13:14:10.920137Z","steps":["trace[1342182906] 'read index received' (duration: 416.277162ms)","trace[1342182906] 'applied index is now lower than readState.Index' (duration: 4.231µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:10.920314Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"416.443105ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:10.920354Z","caller":"traceutil/trace.go:171","msg":"trace[1909206256] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1039; }","duration":"416.523786ms","start":"2026-05-09T13:14:10.503821Z","end":"2026-05-09T13:14:10.920345Z","steps":["trace[1909206256] 'agreement among raft nodes before linearized reading' (duration: 416.420135ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:10.920425Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:10.503807Z","time spent":"416.609887ms","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-05-09T13:14:10.921385Z","caller":"traceutil/trace.go:171","msg":"trace[215170622] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1040; }","duration":"398.994701ms","start":"2026-05-09T13:14:10.522377Z","end":"2026-05-09T13:14:10.921371Z","steps":["trace[215170622] 'process raft request' (duration: 398.748197ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:10.921557Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:10.522350Z","time spent":"399.097032ms","remote":"127.0.0.1:34472","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":104,"response count":0,"response size":65472,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:14:10.922397Z","caller":"traceutil/trace.go:171","msg":"trace[1662213342] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1040; }","duration":"447.910126ms","start":"2026-05-09T13:14:10.474450Z","end":"2026-05-09T13:14:10.922360Z","steps":["trace[1662213342] 'process raft request' (duration: 446.200843ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:10.922587Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:10.474432Z","time spent":"448.063118ms","remote":"127.0.0.1:34472","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":105,"response count":0,"response size":74796,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-05-09T13:14:16.361236Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:16.861471Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:17.362225Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:17.488916Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.196370993s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:17.489029Z","caller":"traceutil/trace.go:171","msg":"trace[2146642736] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1084; }","duration":"1.196488655s","start":"2026-05-09T13:14:16.292514Z","end":"2026-05-09T13:14:17.489003Z","steps":["trace[2146642736] 'range keys from in-memory index tree' (duration: 1.196319752s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:17.488913Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.705375643s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:17.489107Z","caller":"traceutil/trace.go:171","msg":"trace[720911009] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1084; }","duration":"1.705617787s","start":"2026-05-09T13:14:15.783478Z","end":"2026-05-09T13:14:17.489096Z","steps":["trace[720911009] 'range keys from in-memory index tree' (duration: 1.703795853s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:17.489164Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:15.783457Z","time spent":"1.705681167s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-05-09T13:14:17.489159Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.989610877s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:17.489285Z","caller":"traceutil/trace.go:171","msg":"trace[1046538796] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1084; }","duration":"1.989753209s","start":"2026-05-09T13:14:15.499509Z","end":"2026-05-09T13:14:17.489262Z","steps":["trace[1046538796] 'range keys from in-memory index tree' (duration: 1.989543306s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:17.490001Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.933747295s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/statefulsets/\" range_end:\"/registry/statefulsets0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:17.490073Z","caller":"traceutil/trace.go:171","msg":"trace[842678095] range","detail":"{range_begin:/registry/statefulsets/; range_end:/registry/statefulsets0; response_count:0; response_revision:1084; }","duration":"1.933864326s","start":"2026-05-09T13:14:15.556191Z","end":"2026-05-09T13:14:17.490056Z","steps":["trace[842678095] 'count revisions from in-memory index tree' (duration: 1.931455654s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:17.490120Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:15.556168Z","time spent":"1.933938227s","remote":"127.0.0.1:34870","response type":"/etcdserverpb.KV/Range","request count":0,"request size":52,"response count":0,"response size":28,"request content":"key:\"/registry/statefulsets/\" range_end:\"/registry/statefulsets0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:17.863188Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:18.364066Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:18.506294Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000230788s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:18.506535Z","caller":"traceutil/trace.go:171","msg":"trace[2143429735] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.0004403s","start":"2026-05-09T13:14:16.505961Z","end":"2026-05-09T13:14:18.506402Z","steps":["trace[2143429735] 'agreement among raft nodes before linearized reading' (duration: 2.000263208s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:18.506631Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:16.505940Z","time spent":"2.000677404s","remote":"127.0.0.1:34420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/05/09 13:14:18 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-05-09T13:14:18.557888Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1255, vote: 0] ignored MsgPreVote from faf479c52908c230 [logterm: 2, index: 1255] at term 2: lease is not expired (remaining ticks: 10)"} {"level":"warn","ts":"2026-05-09T13:14:18.862496Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"3.065899857s","expected-duration":"1s"} {"level":"warn","ts":"2026-05-09T13:14:18.865137Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:19.366086Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:19.757930Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1255, vote: 0] cast MsgPreVote for faf479c52908c230 [logterm: 2, index: 1255] at term 2"} {"level":"warn","ts":"2026-05-09T13:14:19.867110Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:20.368101Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:20.516142Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000740876s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:20.516253Z","caller":"traceutil/trace.go:171","msg":"trace[89720550] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000897858s","start":"2026-05-09T13:14:18.515319Z","end":"2026-05-09T13:14:20.516217Z","steps":["trace[89720550] 'agreement among raft nodes before linearized reading' (duration: 2.000772106s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:20.516327Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:18.515299Z","time spent":"2.001019109s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/05/09 13:14:20 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-05-09T13:14:20.550070Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261, vote: 0] ignored MsgVote from faf479c52908c230 [logterm: 2, index: 1255] at term 2: lease is not expired (remaining ticks: 10)"} {"level":"warn","ts":"2026-05-09T13:14:20.869121Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441426,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:21.052485Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.999335741s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:21.052597Z","caller":"traceutil/trace.go:171","msg":"trace[598237300] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"4.999826228s","start":"2026-05-09T13:14:16.052749Z","end":"2026-05-09T13:14:21.052575Z","steps":["trace[598237300] 'agreement among raft nodes before linearized reading' (duration: 4.999363692s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:21.052819Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:16.052729Z","time spent":"5.000069251s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} 2026/05/09 13:14:21 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-05-09T13:14:21.236695Z","caller":"traceutil/trace.go:171","msg":"trace[249641042] linearizableReadLoop","detail":"{readStateIndex:1255; appliedIndex:1255; }","duration":"5.376556402s","start":"2026-05-09T13:14:15.860111Z","end":"2026-05-09T13:14:21.236667Z","steps":["trace[249641042] 'read index received' (duration: 5.376545722s)","trace[249641042] 'applied index is now lower than readState.Index' (duration: 9.1µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:21.737949Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:21.883083Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261, vote: 0] ignored MsgPreVote from 3ba9ae53cc4335db [logterm: 2, index: 1261] at term 2: lease is not expired (remaining ticks: 2)"} {"level":"info","ts":"2026-05-09T13:14:21.957286Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261, vote: 0] ignored MsgPreVote from faf479c52908c230 [logterm: 2, index: 1255] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"info","ts":"2026-05-09T13:14:22.208070Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [term: 2] received a MsgVote message with higher term from 3ba9ae53cc4335db [term: 4]"} {"level":"info","ts":"2026-05-09T13:14:22.208155Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became follower at term 4"} {"level":"info","ts":"2026-05-09T13:14:22.208177Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261, vote: 0] cast MsgVote for 3ba9ae53cc4335db [logterm: 2, index: 1261] at term 4"} {"level":"info","ts":"2026-05-09T13:14:22.208198Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: e2c0aeaa63762224 lost leader 3ba9ae53cc4335db at term 4"} {"level":"warn","ts":"2026-05-09T13:14:22.238908Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:22.239020Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-05-09T13:14:22.517280Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000730703s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-05-09T13:14:22.517497Z","caller":"traceutil/trace.go:171","msg":"trace[1653672379] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000954996s","start":"2026-05-09T13:14:20.516466Z","end":"2026-05-09T13:14:22.517421Z","steps":["trace[1653672379] 'agreement among raft nodes before linearized reading' (duration: 2.000752844s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:22.517576Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:20.516443Z","time spent":"2.001116138s","remote":"127.0.0.1:34420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-05-09T13:14:22.739654Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:22.739847Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-05-09T13:14:23.105785Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.999003582s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:23.105852Z","caller":"traceutil/trace.go:171","msg":"trace[1554444409] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999140804s","start":"2026-05-09T13:14:18.106697Z","end":"2026-05-09T13:14:23.105838Z","steps":["trace[1554444409] 'agreement among raft nodes before linearized reading' (duration: 4.999048813s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:23.105892Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:18.106676Z","time spent":"4.999208425s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} 2026/05/09 13:14:23 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-05-09T13:14:23.240823Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:23.240936Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"info","ts":"2026-05-09T13:14:23.582374Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261, vote: 3ba9ae53cc4335db] cast MsgPreVote for 3ba9ae53cc4335db [logterm: 2, index: 1261] at term 4"} {"level":"warn","ts":"2026-05-09T13:14:23.741257Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:23.741399Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-05-09T13:14:24.241530Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:24.241681Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-05-09T13:14:24.527797Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000663251s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:24.527927Z","caller":"traceutil/trace.go:171","msg":"trace[1828705259] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000861114s","start":"2026-05-09T13:14:22.527031Z","end":"2026-05-09T13:14:24.527892Z","steps":["trace[1828705259] 'agreement among raft nodes before linearized reading' (duration: 2.000712922s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:24.528969Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:22.527000Z","time spent":"2.001911948s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/05/09 13:14:24 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-05-09T13:14:24.742670Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:24.742817Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"info","ts":"2026-05-09T13:14:25.083627Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261, vote: 3ba9ae53cc4335db] cast MsgPreVote for 3ba9ae53cc4335db [logterm: 2, index: 1261] at term 4"} {"level":"warn","ts":"2026-05-09T13:14:25.243198Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:25.243292Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-05-09T13:14:25.619225Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"3.410880978s","expected-duration":"1s"} {"level":"warn","ts":"2026-05-09T13:14:25.744026Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:25.744177Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-05-09T13:14:25.899285Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:18.898737Z","time spent":"7.000538744s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-05-09T13:14:26.062438Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:19.061697Z","time spent":"7.00073292s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-05-09T13:14:26.244687Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:26.244808Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-05-09T13:14:26.528890Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000100326s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-05-09T13:14:26.529013Z","caller":"traceutil/trace.go:171","msg":"trace[1357450059] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000304649s","start":"2026-05-09T13:14:24.528681Z","end":"2026-05-09T13:14:26.528985Z","steps":["trace[1357450059] 'agreement among raft nodes before linearized reading' (duration: 2.000147727s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:26.529076Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:24.528633Z","time spent":"2.00043161s","remote":"127.0.0.1:34420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/05/09 13:14:26 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-05-09T13:14:26.744983Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:26.745123Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-05-09T13:14:27.245467Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:27.245697Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 no leader at term 4; dropping index reading msg"} {"level":"info","ts":"2026-05-09T13:14:27.252858Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 is starting a new election at term 4"} {"level":"info","ts":"2026-05-09T13:14:27.252974Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became pre-candidate at term 4"} {"level":"info","ts":"2026-05-09T13:14:27.252998Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 received MsgPreVoteResp from e2c0aeaa63762224 at term 4"} {"level":"info","ts":"2026-05-09T13:14:27.253032Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261] sent MsgPreVote request to 3ba9ae53cc4335db at term 4"} {"level":"info","ts":"2026-05-09T13:14:27.253054Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261] sent MsgPreVote request to faf479c52908c230 at term 4"} {"level":"warn","ts":"2026-05-09T13:14:27.746491Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441429,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:27.782783Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.999897769s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-05-09T13:14:27.782886Z","caller":"traceutil/trace.go:171","msg":"trace[1551383008] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000103351s","start":"2026-05-09T13:14:25.782760Z","end":"2026-05-09T13:14:27.782863Z","steps":["trace[1551383008] 'agreement among raft nodes before linearized reading' (duration: 1.999941049s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:27.782946Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:25.782732Z","time spent":"2.000206453s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-05-09T13:14:28.237793Z","caller":"etcdserver/v3_server.go:932","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"} {"level":"warn","ts":"2026-05-09T13:14:28.238040Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"10.647443319s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-05-09T13:14:28.238143Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"8.224827397s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/\" range_end:\"/registry/configmaps0\" count_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-05-09T13:14:28.238165Z","caller":"traceutil/trace.go:171","msg":"trace[1063292596] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; }","duration":"10.647631271s","start":"2026-05-09T13:14:17.590513Z","end":"2026-05-09T13:14:28.238144Z","steps":["trace[1063292596] 'agreement among raft nodes before linearized reading' (duration: 10.647470919s)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:28.238184Z","caller":"traceutil/trace.go:171","msg":"trace[1729207059] range","detail":"{range_begin:/registry/configmaps/; range_end:/registry/configmaps0; }","duration":"8.224898488s","start":"2026-05-09T13:14:20.013266Z","end":"2026-05-09T13:14:28.238165Z","steps":["trace[1729207059] 'agreement among raft nodes before linearized reading' (duration: 8.224872338s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.238171Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"10.74889831s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-05-09T13:14:28.238159Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"11.950982158s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-05-09T13:14:28.238188Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"9.72879789s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18ade7f06215ec40\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-05-09T13:14:28.238242Z","caller":"traceutil/trace.go:171","msg":"trace[418406704] range","detail":"{range_begin:; range_end:; }","duration":"10.749137113s","start":"2026-05-09T13:14:17.489081Z","end":"2026-05-09T13:14:28.238218Z","steps":["trace[418406704] 'agreement among raft nodes before linearized reading' (duration: 10.74889563s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.238258Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"11.647410217s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.109\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-05-09T13:14:28.238266Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"10.576504991s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-05-09T13:14:28.238283Z","caller":"traceutil/trace.go:171","msg":"trace[101674137] range","detail":"{range_begin:/registry/masterleases/199.204.45.109; range_end:; }","duration":"11.647473417s","start":"2026-05-09T13:14:16.590803Z","end":"2026-05-09T13:14:28.238277Z","steps":["trace[101674137] 'agreement among raft nodes before linearized reading' (duration: 11.647449196s)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:28.238275Z","caller":"traceutil/trace.go:171","msg":"trace[1965856731] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; }","duration":"11.951188291s","start":"2026-05-09T13:14:16.287064Z","end":"2026-05-09T13:14:28.238252Z","steps":["trace[1965856731] 'agreement among raft nodes before linearized reading' (duration: 11.951002729s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.238315Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:16.590782Z","time spent":"11.647528497s","remote":"127.0.0.1:34440","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":0,"response size":0,"request content":"key:\"/registry/masterleases/199.204.45.109\" limit:1 "} {"level":"info","ts":"2026-05-09T13:14:28.238314Z","caller":"traceutil/trace.go:171","msg":"trace[1254028180] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; }","duration":"10.576602932s","start":"2026-05-09T13:14:17.661702Z","end":"2026-05-09T13:14:28.238305Z","steps":["trace[1254028180] 'agreement among raft nodes before linearized reading' (duration: 10.576556642s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.238325Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"12.046066478s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-kv7qd\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-05-09T13:14:28.238341Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:16.287042Z","time spent":"11.951287762s","remote":"127.0.0.1:34952","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":0,"response size":0,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:28.238344Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:17.661674Z","time spent":"10.576662453s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"error","ts":"2026-05-09T13:14:28.238321Z","caller":"etcdhttp/health.go:370","msg":"Health check error","path":"/readyz","reason":"[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: request timed out\n[+]non_learner ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHttpEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:370\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-05-09T13:14:28.238229Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:17.590498Z","time spent":"10.647719002s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:28.238384Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"12.121656431s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicies/\" range_end:\"/registry/validatingadmissionpolicies0\" count_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-05-09T13:14:28.238424Z","caller":"traceutil/trace.go:171","msg":"trace[34950265] range","detail":"{range_begin:/registry/validatingadmissionpolicies/; range_end:/registry/validatingadmissionpolicies0; }","duration":"12.121722281s","start":"2026-05-09T13:14:16.116685Z","end":"2026-05-09T13:14:28.238407Z","steps":["trace[34950265] 'agreement among raft nodes before linearized reading' (duration: 12.121690161s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.238486Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:16.116666Z","time spent":"12.121802002s","remote":"127.0.0.1:34932","response type":"/etcdserverpb.KV/Range","request count":0,"request size":82,"response count":0,"response size":0,"request content":"key:\"/registry/validatingadmissionpolicies/\" range_end:\"/registry/validatingadmissionpolicies0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:28.238227Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:20.013235Z","time spent":"8.224980809s","remote":"127.0.0.1:34508","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/configmaps/\" range_end:\"/registry/configmaps0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:28.238063Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"9.578199622s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/networkpolicies/\" range_end:\"/registry/networkpolicies0\" count_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-05-09T13:14:28.239453Z","caller":"traceutil/trace.go:171","msg":"trace[735747228] range","detail":"{range_begin:/registry/networkpolicies/; range_end:/registry/networkpolicies0; }","duration":"9.57964636s","start":"2026-05-09T13:14:18.659772Z","end":"2026-05-09T13:14:28.239418Z","steps":["trace[735747228] 'agreement among raft nodes before linearized reading' (duration: 9.578240092s)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:28.238285Z","caller":"traceutil/trace.go:171","msg":"trace[331283407] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18ade7f06215ec40; range_end:; }","duration":"9.728949702s","start":"2026-05-09T13:14:18.509318Z","end":"2026-05-09T13:14:28.238268Z","steps":["trace[331283407] 'agreement among raft nodes before linearized reading' (duration: 9.72881982s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.239519Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:18.659749Z","time spent":"9.579752781s","remote":"127.0.0.1:34688","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":0,"response size":0,"request content":"key:\"/registry/networkpolicies/\" range_end:\"/registry/networkpolicies0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:28.239544Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:18.509300Z","time spent":"9.730221337s","remote":"127.0.0.1:34490","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":0,"response size":0,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18ade7f06215ec40\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:28.238122Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.089851931s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/flowschemas/\" range_end:\"/registry/flowschemas0\" count_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-05-09T13:14:28.239628Z","caller":"traceutil/trace.go:171","msg":"trace[706016688] range","detail":"{range_begin:/registry/flowschemas/; range_end:/registry/flowschemas0; }","duration":"7.09135388s","start":"2026-05-09T13:14:21.148260Z","end":"2026-05-09T13:14:28.239614Z","steps":["trace[706016688] 'agreement among raft nodes before linearized reading' (duration: 7.089851811s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.239700Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:21.148200Z","time spent":"7.091486822s","remote":"127.0.0.1:34840","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":0,"response size":0,"request content":"key:\"/registry/flowschemas/\" range_end:\"/registry/flowschemas0\" count_only:true "} {"level":"info","ts":"2026-05-09T13:14:28.238357Z","caller":"traceutil/trace.go:171","msg":"trace[680135122] range","detail":"{range_begin:/registry/pods/kube-system/cilium-kv7qd; range_end:; }","duration":"12.04614084s","start":"2026-05-09T13:14:16.192207Z","end":"2026-05-09T13:14:28.238348Z","steps":["trace[680135122] 'agreement among raft nodes before linearized reading' (duration: 12.046106269s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.239754Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:16.192188Z","time spent":"12.047558687s","remote":"127.0.0.1:34580","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":0,"response size":0,"request content":"key:\"/registry/pods/kube-system/cilium-kv7qd\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:28.539200Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000322802s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:28.539301Z","caller":"traceutil/trace.go:171","msg":"trace[694511632] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000523085s","start":"2026-05-09T13:14:26.538749Z","end":"2026-05-09T13:14:28.539272Z","steps":["trace[694511632] 'agreement among raft nodes before linearized reading' (duration: 2.000367383s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:28.539354Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:26.538722Z","time spent":"2.000621916s","remote":"127.0.0.1:34420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/05/09 13:14:28 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-05-09T13:14:28.738324Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441432,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:29.052750Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 is starting a new election at term 4"} {"level":"info","ts":"2026-05-09T13:14:29.052825Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became pre-candidate at term 4"} {"level":"info","ts":"2026-05-09T13:14:29.052850Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 received MsgPreVoteResp from e2c0aeaa63762224 at term 4"} {"level":"info","ts":"2026-05-09T13:14:29.052878Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261] sent MsgPreVote request to 3ba9ae53cc4335db at term 4"} {"level":"info","ts":"2026-05-09T13:14:29.052896Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261] sent MsgPreVote request to faf479c52908c230 at term 4"} {"level":"warn","ts":"2026-05-09T13:14:29.239039Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441432,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:29.300881Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.999205337s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-05-09T13:14:29.301002Z","caller":"traceutil/trace.go:171","msg":"trace[1848853358] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"4.999421839s","start":"2026-05-09T13:14:24.301556Z","end":"2026-05-09T13:14:29.300978Z","steps":["trace[1848853358] 'agreement among raft nodes before linearized reading' (duration: 4.999254857s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:29.301073Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:24.301528Z","time spent":"4.999532711s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} 2026/05/09 13:14:29 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-05-09T13:14:29.740288Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441432,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:29.920036Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 received MsgPreVoteResp from faf479c52908c230 at term 4"} {"level":"info","ts":"2026-05-09T13:14:29.920137Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 has received 2 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-05-09T13:14:29.920165Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became candidate at term 5"} {"level":"info","ts":"2026-05-09T13:14:29.920178Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 received MsgVoteResp from e2c0aeaa63762224 at term 5"} {"level":"info","ts":"2026-05-09T13:14:29.920207Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261] sent MsgVote request to 3ba9ae53cc4335db at term 5"} {"level":"info","ts":"2026-05-09T13:14:29.920223Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 2, index: 1261] sent MsgVote request to faf479c52908c230 at term 5"} {"level":"warn","ts":"2026-05-09T13:14:30.032001Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.531902663s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-05-09T13:14:30.032069Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"14.171903773s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/\" range_end:\"/registry/services/endpoints0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-05-09T13:14:30.032124Z","caller":"traceutil/trace.go:171","msg":"trace[19146499] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1085; }","duration":"4.532045085s","start":"2026-05-09T13:14:25.500051Z","end":"2026-05-09T13:14:30.032096Z","steps":["trace[19146499] 'range keys from in-memory index tree' (duration: 4.531808112s)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:30.032184Z","caller":"traceutil/trace.go:171","msg":"trace[1431066362] range","detail":"{range_begin:/registry/services/endpoints/; range_end:/registry/services/endpoints0; response_count:0; response_revision:1085; }","duration":"14.172075176s","start":"2026-05-09T13:14:15.860080Z","end":"2026-05-09T13:14:30.032155Z","steps":["trace[1431066362] 'agreement among raft nodes before linearized reading' (duration: 5.376682244s)","trace[1431066362] 'count revisions from in-memory index tree' (duration: 8.795186248s)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:30.032232Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:15.860064Z","time spent":"14.172159237s","remote":"127.0.0.1:34556","response type":"/etcdserverpb.KV/Range","request count":0,"request size":64,"response count":2,"response size":30,"request content":"key:\"/registry/services/endpoints/\" range_end:\"/registry/services/endpoints0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:30.240500Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441432,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:30.445562Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 received MsgVoteResp from faf479c52908c230 at term 5"} {"level":"info","ts":"2026-05-09T13:14:30.445611Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 has received 2 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-05-09T13:14:30.445625Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 became leader at term 5"} {"level":"info","ts":"2026-05-09T13:14:30.445637Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: e2c0aeaa63762224 elected leader e2c0aeaa63762224 at term 5"} {"level":"warn","ts":"2026-05-09T13:14:30.449066Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.376441674s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-05-09T13:14:30.449130Z","caller":"traceutil/trace.go:171","msg":"trace[731622656] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"3.376575005s","start":"2026-05-09T13:14:27.072543Z","end":"2026-05-09T13:14:30.449118Z","steps":["trace[731622656] 'agreement among raft nodes before linearized reading' (duration: 3.376493824s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:30.449169Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:27.072512Z","time spent":"3.376648106s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:30.449204Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.637236798s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/certificatesigningrequests/\" range_end:\"/registry/certificatesigningrequests0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-05-09T13:14:30.449222Z","caller":"traceutil/trace.go:171","msg":"trace[529452453] range","detail":"{range_begin:/registry/certificatesigningrequests/; range_end:/registry/certificatesigningrequests0; }","duration":"3.637303038s","start":"2026-05-09T13:14:26.811912Z","end":"2026-05-09T13:14:30.449216Z","steps":["trace[529452453] 'agreement among raft nodes before linearized reading' (duration: 3.637283198s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:30.449242Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:26.811877Z","time spent":"3.637358019s","remote":"127.0.0.1:34662","response type":"/etcdserverpb.KV/Range","request count":0,"request size":80,"response count":0,"response size":0,"request content":"key:\"/registry/certificatesigningrequests/\" range_end:\"/registry/certificatesigningrequests0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:30.449268Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.303489682s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/replicasets/\" range_end:\"/registry/replicasets0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-05-09T13:14:30.449284Z","caller":"traceutil/trace.go:171","msg":"trace[856952990] range","detail":"{range_begin:/registry/replicasets/; range_end:/registry/replicasets0; }","duration":"4.303535944s","start":"2026-05-09T13:14:26.145743Z","end":"2026-05-09T13:14:30.449279Z","steps":["trace[856952990] 'agreement among raft nodes before linearized reading' (duration: 4.303518403s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:30.449302Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:26.145725Z","time spent":"4.303572094s","remote":"127.0.0.1:34896","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":0,"response size":0,"request content":"key:\"/registry/replicasets/\" range_end:\"/registry/replicasets0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:30.449315Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.535696822s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingwebhookconfigurations/\" range_end:\"/registry/validatingwebhookconfigurations0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-05-09T13:14:30.449500Z","caller":"traceutil/trace.go:171","msg":"trace[1491214721] range","detail":"{range_begin:/registry/validatingwebhookconfigurations/; range_end:/registry/validatingwebhookconfigurations0; }","duration":"6.535946794s","start":"2026-05-09T13:14:23.913543Z","end":"2026-05-09T13:14:30.449490Z","steps":["trace[1491214721] 'agreement among raft nodes before linearized reading' (duration: 6.535745372s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:30.449525Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:23.913518Z","time spent":"6.536000166s","remote":"127.0.0.1:34918","response type":"/etcdserverpb.KV/Range","request count":0,"request size":90,"response count":0,"response size":0,"request content":"key:\"/registry/validatingwebhookconfigurations/\" range_end:\"/registry/validatingwebhookconfigurations0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:30.449757Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"8.123884921s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csidrivers/\" range_end:\"/registry/csidrivers0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-05-09T13:14:30.449780Z","caller":"traceutil/trace.go:171","msg":"trace[1622925570] range","detail":"{range_begin:/registry/csidrivers/; range_end:/registry/csidrivers0; }","duration":"8.123953842s","start":"2026-05-09T13:14:22.325821Z","end":"2026-05-09T13:14:30.449775Z","steps":["trace[1622925570] 'agreement among raft nodes before linearized reading' (duration: 8.123927762s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:30.449795Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:22.325796Z","time spent":"8.123994772s","remote":"127.0.0.1:34822","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/csidrivers/\" range_end:\"/registry/csidrivers0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:30.449817Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"8.47092268s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumes/\" range_end:\"/registry/persistentvolumes0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-05-09T13:14:30.449857Z","caller":"traceutil/trace.go:171","msg":"trace[437276863] range","detail":"{range_begin:/registry/persistentvolumes/; range_end:/registry/persistentvolumes0; }","duration":"8.471016112s","start":"2026-05-09T13:14:21.978837Z","end":"2026-05-09T13:14:30.449853Z","steps":["trace[437276863] 'agreement among raft nodes before linearized reading' (duration: 8.470975911s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:30.449871Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:21.978811Z","time spent":"8.471056582s","remote":"127.0.0.1:34542","response type":"/etcdserverpb.KV/Range","request count":0,"request size":62,"response count":0,"response size":0,"request content":"key:\"/registry/persistentvolumes/\" range_end:\"/registry/persistentvolumes0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:30.449927Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"8.626760666s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/prioritylevelconfigurations/\" range_end:\"/registry/prioritylevelconfigurations0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-05-09T13:14:30.449991Z","caller":"traceutil/trace.go:171","msg":"trace[856494858] range","detail":"{range_begin:/registry/prioritylevelconfigurations/; range_end:/registry/prioritylevelconfigurations0; }","duration":"8.626900667s","start":"2026-05-09T13:14:21.823081Z","end":"2026-05-09T13:14:30.449981Z","steps":["trace[856494858] 'agreement among raft nodes before linearized reading' (duration: 8.626814966s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:30.450020Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:21.823051Z","time spent":"8.626960539s","remote":"127.0.0.1:34856","response type":"/etcdserverpb.KV/Range","request count":0,"request size":82,"response count":0,"response size":0,"request content":"key:\"/registry/prioritylevelconfigurations/\" range_end:\"/registry/prioritylevelconfigurations0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:30.548175Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.999895723s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-05-09T13:14:30.548246Z","caller":"traceutil/trace.go:171","msg":"trace[817958785] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000028726s","start":"2026-05-09T13:14:28.548201Z","end":"2026-05-09T13:14:30.548230Z","steps":["trace[817958785] 'agreement among raft nodes before linearized reading' (duration: 1.999930234s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:30.548288Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:28.548159Z","time spent":"2.000115107s","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-05-09T13:14:30.949233Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441433,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-05-09T13:14:31.007271Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"faf479c52908c230","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"325.902343ms"} {"level":"warn","ts":"2026-05-09T13:14:31.007402Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3ba9ae53cc4335db","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"326.051865ms"} {"level":"warn","ts":"2026-05-09T13:14:31.450501Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2460265074559441433,"retry-timeout":"500ms"} {"level":"info","ts":"2026-05-09T13:14:31.470341Z","caller":"etcdserver/v3_server.go:912","msg":"first commit in current term: resending ReadIndex request"} {"level":"info","ts":"2026-05-09T13:14:31.470402Z","caller":"traceutil/trace.go:171","msg":"trace[1354840460] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1091; }","duration":"5.564348849s","start":"2026-05-09T13:14:25.906000Z","end":"2026-05-09T13:14:31.470349Z","steps":["trace[1354840460] 'process raft request' (duration: 5.564287059s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:31.470598Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:25.905972Z","time spent":"5.564523671s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":69,"response count":0,"response size":726,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:14:31.470680Z","caller":"traceutil/trace.go:171","msg":"trace[327645530] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1091; }","duration":"5.402128724s","start":"2026-05-09T13:14:26.068455Z","end":"2026-05-09T13:14:31.470584Z","steps":["trace[327645530] 'process raft request' (duration: 5.401864261s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:31.470849Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:26.068422Z","time spent":"5.402323897s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":49,"response count":0,"response size":585,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:14:31.698295Z","caller":"traceutil/trace.go:171","msg":"trace[974555132] linearizableReadLoop","detail":"{readStateIndex:1264; appliedIndex:1264; }","duration":"1.249290819s","start":"2026-05-09T13:14:30.448957Z","end":"2026-05-09T13:14:31.698248Z","steps":["trace[974555132] 'read index received' (duration: 1.249064336s)","trace[974555132] 'applied index is now lower than readState.Index' (duration: 225.153µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:31.766100Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.526393812s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:31.766191Z","caller":"traceutil/trace.go:171","msg":"trace[1580324622] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1091; }","duration":"3.526498854s","start":"2026-05-09T13:14:28.239668Z","end":"2026-05-09T13:14:31.766166Z","steps":["trace[1580324622] 'agreement among raft nodes before linearized reading' (duration: 3.458716608s)","trace[1580324622] 'range keys from in-memory index tree' (duration: 67.660394ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:31.766243Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.814915568s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:31.766355Z","caller":"traceutil/trace.go:171","msg":"trace[1392876834] range","detail":"{range_begin:/registry/horizontalpodautoscalers/; range_end:/registry/horizontalpodautoscalers0; response_count:0; response_revision:1091; }","duration":"1.815106751s","start":"2026-05-09T13:14:29.951216Z","end":"2026-05-09T13:14:31.766322Z","steps":["trace[1392876834] 'agreement among raft nodes before linearized reading' (duration: 1.747002781s)","trace[1392876834] 'count revisions from in-memory index tree' (duration: 67.939008ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:31.766358Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.399881246s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions/\" range_end:\"/registry/minions0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"warn","ts":"2026-05-09T13:14:31.766456Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.458307442s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/\" range_end:\"/registry/apiregistration.k8s.io/apiservices0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-05-09T13:14:31.766456Z","caller":"traceutil/trace.go:171","msg":"trace[476865857] range","detail":"{range_begin:/registry/minions/; range_end:/registry/minions0; response_count:0; response_revision:1091; }","duration":"2.400056099s","start":"2026-05-09T13:14:29.366374Z","end":"2026-05-09T13:14:31.766430Z","steps":["trace[476865857] 'agreement among raft nodes before linearized reading' (duration: 2.331971289s)","trace[476865857] 'count revisions from in-memory index tree' (duration: 67.936238ms)"],"step_count":2} {"level":"info","ts":"2026-05-09T13:14:31.766485Z","caller":"traceutil/trace.go:171","msg":"trace[2109013045] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/; range_end:/registry/apiregistration.k8s.io/apiservices0; response_count:0; response_revision:1091; }","duration":"1.458346152s","start":"2026-05-09T13:14:30.308126Z","end":"2026-05-09T13:14:31.766472Z","steps":["trace[2109013045] 'agreement among raft nodes before linearized reading' (duration: 1.39004675s)","trace[2109013045] 'count revisions from in-memory index tree' (duration: 68.248532ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:31.766510Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:30.308096Z","time spent":"1.458408434s","remote":"127.0.0.1:34958","response type":"/etcdserverpb.KV/Range","request count":0,"request size":96,"response count":26,"response size":30,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices/\" range_end:\"/registry/apiregistration.k8s.io/apiservices0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:31.766526Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:29.366341Z","time spent":"2.40016204s","remote":"127.0.0.1:34570","response type":"/etcdserverpb.KV/Range","request count":0,"request size":42,"response count":3,"response size":30,"request content":"key:\"/registry/minions/\" range_end:\"/registry/minions0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:31.766556Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.521386627s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18ade7f06215ec40\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"warn","ts":"2026-05-09T13:14:31.766549Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.52555573s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.109\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-05-09T13:14:31.766603Z","caller":"traceutil/trace.go:171","msg":"trace[835807829] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18ade7f06215ec40; range_end:; response_count:1; response_revision:1091; }","duration":"3.521464468s","start":"2026-05-09T13:14:28.245122Z","end":"2026-05-09T13:14:31.766586Z","steps":["trace[835807829] 'agreement among raft nodes before linearized reading' (duration: 3.453195786s)","trace[835807829] 'range keys from in-memory index tree' (duration: 68.184091ms)"],"step_count":2} {"level":"info","ts":"2026-05-09T13:14:31.766628Z","caller":"traceutil/trace.go:171","msg":"trace[1140892045] range","detail":"{range_begin:/registry/masterleases/199.204.45.109; range_end:; response_count:1; response_revision:1091; }","duration":"3.525676602s","start":"2026-05-09T13:14:28.240933Z","end":"2026-05-09T13:14:31.766610Z","steps":["trace[1140892045] 'agreement among raft nodes before linearized reading' (duration: 3.45741994s)","trace[1140892045] 'range keys from in-memory index tree' (duration: 68.137051ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:31.766682Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:28.240909Z","time spent":"3.525759503s","remote":"127.0.0.1:34440","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.109\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:31.766638Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:28.245102Z","time spent":"3.521527768s","remote":"127.0.0.1:34490","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":817,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18ade7f06215ec40\" limit:1 "} {"level":"warn","ts":"2026-05-09T13:14:31.766402Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:29.951182Z","time spent":"1.815211861s","remote":"127.0.0.1:34612","response type":"/etcdserverpb.KV/Range","request count":0,"request size":76,"response count":0,"response size":28,"request content":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:31.766243Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.521378177s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 ","response":"range_response_count:1 size:7619"} {"level":"info","ts":"2026-05-09T13:14:31.767035Z","caller":"traceutil/trace.go:171","msg":"trace[679777114] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-2; range_end:; response_count:1; response_revision:1091; }","duration":"3.522235159s","start":"2026-05-09T13:14:28.244770Z","end":"2026-05-09T13:14:31.767005Z","steps":["trace[679777114] 'agreement among raft nodes before linearized reading' (duration: 3.453560431s)","trace[679777114] 'range keys from in-memory index tree' (duration: 67.775526ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:31.767064Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.218085485s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-05-09T13:14:31.767094Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:28.244740Z","time spent":"3.52233753s","remote":"127.0.0.1:34580","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":1,"response size":7642,"request content":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 "} {"level":"info","ts":"2026-05-09T13:14:31.767120Z","caller":"traceutil/trace.go:171","msg":"trace[2121153651] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1093; }","duration":"1.218156155s","start":"2026-05-09T13:14:30.548953Z","end":"2026-05-09T13:14:31.767109Z","steps":["trace[2121153651] 'agreement among raft nodes before linearized reading' (duration: 1.218084645s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:31.767676Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:30.548939Z","time spent":"1.218726433s","remote":"127.0.0.1:34420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-05-09T13:14:31.767801Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.070642967s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/\" range_end:\"/registry/endpointslices0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"warn","ts":"2026-05-09T13:14:31.767816Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"292.696262ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/apiserver-zjy5m4qienmmbtxvgklj3tgz6i\" limit:1 ","response":"range_response_count:1 size:692"} {"level":"info","ts":"2026-05-09T13:14:31.767850Z","caller":"traceutil/trace.go:171","msg":"trace[1127574013] range","detail":"{range_begin:/registry/endpointslices/; range_end:/registry/endpointslices0; response_count:0; response_revision:1093; }","duration":"1.070696508s","start":"2026-05-09T13:14:30.697139Z","end":"2026-05-09T13:14:31.767835Z","steps":["trace[1127574013] 'agreement among raft nodes before linearized reading' (duration: 1.070609216s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:31.767887Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:30.697125Z","time spent":"1.070752158s","remote":"127.0.0.1:34674","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":2,"response size":30,"request content":"key:\"/registry/endpointslices/\" range_end:\"/registry/endpointslices0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:31.767894Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.094314473s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-05-09T13:14:31.767938Z","caller":"traceutil/trace.go:171","msg":"trace[159754809] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1093; }","duration":"1.094424915s","start":"2026-05-09T13:14:30.673502Z","end":"2026-05-09T13:14:31.767927Z","steps":["trace[159754809] 'agreement among raft nodes before linearized reading' (duration: 1.094335504s)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:31.767993Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:30.673479Z","time spent":"1.094504615s","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-05-09T13:14:31.768233Z","caller":"traceutil/trace.go:171","msg":"trace[1943681772] range","detail":"{range_begin:/registry/leases/kube-system/apiserver-zjy5m4qienmmbtxvgklj3tgz6i; range_end:; response_count:1; response_revision:1093; }","duration":"292.791184ms","start":"2026-05-09T13:14:31.475063Z","end":"2026-05-09T13:14:31.767854Z","steps":["trace[1943681772] 'agreement among raft nodes before linearized reading' (duration: 292.628512ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:31.768562Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"292.894574ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-node-lease/controller-2\" limit:1 ","response":"range_response_count:1 size:551"} {"level":"info","ts":"2026-05-09T13:14:31.768630Z","caller":"traceutil/trace.go:171","msg":"trace[68211349] range","detail":"{range_begin:/registry/leases/kube-node-lease/controller-2; range_end:; response_count:1; response_revision:1093; }","duration":"293.002065ms","start":"2026-05-09T13:14:31.475619Z","end":"2026-05-09T13:14:31.768621Z","steps":["trace[68211349] 'agreement among raft nodes before linearized reading' (duration: 292.904184ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:32.030093Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"faf479c52908c230","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"61.322175ms"} {"level":"warn","ts":"2026-05-09T13:14:32.030165Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3ba9ae53cc4335db","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"61.404346ms"} {"level":"info","ts":"2026-05-09T13:14:32.030142Z","caller":"traceutil/trace.go:171","msg":"trace[33064969] linearizableReadLoop","detail":"{readStateIndex:1267; appliedIndex:1267; }","duration":"261.541957ms","start":"2026-05-09T13:14:31.768569Z","end":"2026-05-09T13:14:32.030111Z","steps":["trace[33064969] 'read index received' (duration: 261.525887ms)","trace[33064969] 'applied index is now lower than readState.Index' (duration: 13.93µs)"],"step_count":2} {"level":"info","ts":"2026-05-09T13:14:32.059136Z","caller":"traceutil/trace.go:171","msg":"trace[720138114] transaction","detail":"{read_only:false; response_revision:1094; number_of_response:1; }","duration":"288.947841ms","start":"2026-05-09T13:14:31.770167Z","end":"2026-05-09T13:14:32.059114Z","steps":["trace[720138114] 'process raft request' (duration: 260.11866ms)","trace[720138114] 'compare' (duration: 28.6988ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:32.059107Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"290.503952ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:32.059298Z","caller":"traceutil/trace.go:171","msg":"trace[618928088] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1093; }","duration":"290.725415ms","start":"2026-05-09T13:14:31.768563Z","end":"2026-05-09T13:14:32.059289Z","steps":["trace[618928088] 'agreement among raft nodes before linearized reading' (duration: 261.7048ms)","trace[618928088] 'range keys from in-memory index tree' (duration: 28.767541ms)"],"step_count":2} {"level":"info","ts":"2026-05-09T13:14:32.276403Z","caller":"traceutil/trace.go:171","msg":"trace[40377021] transaction","detail":"{read_only:false; response_revision:1096; number_of_response:1; }","duration":"501.748006ms","start":"2026-05-09T13:14:31.774631Z","end":"2026-05-09T13:14:32.276379Z","steps":["trace[40377021] 'process raft request' (duration: 501.665525ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:32.276926Z","caller":"traceutil/trace.go:171","msg":"trace[1944214774] transaction","detail":"{read_only:false; response_revision:1097; number_of_response:1; }","duration":"501.327861ms","start":"2026-05-09T13:14:31.775581Z","end":"2026-05-09T13:14:32.276909Z","steps":["trace[1944214774] 'process raft request' (duration: 500.758933ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:32.276997Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:31.775563Z","time spent":"501.396712ms","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-05-09T13:14:32.276983Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:31.774606Z","time spent":"502.282165ms","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:14:32.277034Z","caller":"traceutil/trace.go:171","msg":"trace[1230722244] transaction","detail":"{read_only:false; response_revision:1098; number_of_response:1; }","duration":"496.5548ms","start":"2026-05-09T13:14:31.780450Z","end":"2026-05-09T13:14:32.277004Z","steps":["trace[1230722244] 'process raft request' (duration: 496.306317ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:32.276437Z","caller":"traceutil/trace.go:171","msg":"trace[1496414407] transaction","detail":"{read_only:false; response_revision:1095; number_of_response:1; }","duration":"502.991882ms","start":"2026-05-09T13:14:31.773411Z","end":"2026-05-09T13:14:32.276403Z","steps":["trace[1496414407] 'process raft request' (duration: 481.726459ms)","trace[1496414407] 'compare' (duration: 21.0053ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:32.277213Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:31.780429Z","time spent":"496.687771ms","remote":"127.0.0.1:34580","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7796,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-05-09T13:14:32.277336Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:31.773397Z","time spent":"503.848844ms","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-05-09T13:14:32.295375Z","caller":"traceutil/trace.go:171","msg":"trace[1593314735] linearizableReadLoop","detail":"{readStateIndex:1268; appliedIndex:1274; }","duration":"265.151889ms","start":"2026-05-09T13:14:32.030206Z","end":"2026-05-09T13:14:32.295358Z","steps":["trace[1593314735] 'read index received' (duration: 265.147259ms)","trace[1593314735] 'applied index is now lower than readState.Index' (duration: 3.62µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:32.295492Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"518.851237ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:32.295528Z","caller":"traceutil/trace.go:171","msg":"trace[2066825689] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1099; }","duration":"519.562956ms","start":"2026-05-09T13:14:31.775954Z","end":"2026-05-09T13:14:32.295517Z","steps":["trace[2066825689] 'agreement among raft nodes before linearized reading' (duration: 519.486985ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:32.295557Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:31.775944Z","time spent":"519.605947ms","remote":"127.0.0.1:34416","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-05-09T13:14:32.295649Z","caller":"traceutil/trace.go:171","msg":"trace[2109895804] transaction","detail":"{read_only:false; response_revision:1100; number_of_response:1; }","duration":"233.141297ms","start":"2026-05-09T13:14:32.062491Z","end":"2026-05-09T13:14:32.295633Z","steps":["trace[2109895804] 'process raft request' (duration: 233.049015ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:32.295730Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"515.042098ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/runtimeclasses/\" range_end:\"/registry/runtimeclasses0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:32.295813Z","caller":"traceutil/trace.go:171","msg":"trace[1859360044] range","detail":"{range_begin:/registry/runtimeclasses/; range_end:/registry/runtimeclasses0; response_count:0; response_revision:1100; }","duration":"515.17387ms","start":"2026-05-09T13:14:31.780621Z","end":"2026-05-09T13:14:32.295794Z","steps":["trace[1859360044] 'agreement among raft nodes before linearized reading' (duration: 514.986737ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:32.295887Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:31.780603Z","time spent":"515.2462ms","remote":"127.0.0.1:34730","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":28,"request content":"key:\"/registry/runtimeclasses/\" range_end:\"/registry/runtimeclasses0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:32.296286Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"373.254366ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllers/\" range_end:\"/registry/controllers0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:32.296368Z","caller":"traceutil/trace.go:171","msg":"trace[1856922331] range","detail":"{range_begin:/registry/controllers/; range_end:/registry/controllers0; response_count:0; response_revision:1100; }","duration":"373.405419ms","start":"2026-05-09T13:14:31.922939Z","end":"2026-05-09T13:14:32.296345Z","steps":["trace[1856922331] 'agreement among raft nodes before linearized reading' (duration: 372.639918ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:32.296451Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:31.922915Z","time spent":"373.519871ms","remote":"127.0.0.1:34598","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":0,"response size":28,"request content":"key:\"/registry/controllers/\" range_end:\"/registry/controllers0\" count_only:true "} {"level":"warn","ts":"2026-05-09T13:14:32.296575Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"233.451212ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18ade7f06215ec40\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-05-09T13:14:32.297595Z","caller":"traceutil/trace.go:171","msg":"trace[439723589] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18ade7f06215ec40; range_end:; response_count:1; response_revision:1100; }","duration":"234.526845ms","start":"2026-05-09T13:14:32.063044Z","end":"2026-05-09T13:14:32.297571Z","steps":["trace[439723589] 'agreement among raft nodes before linearized reading' (duration: 233.394361ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:32.300931Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"e2c0aeaa63762224 [logterm: 5, index: 1275, vote: e2c0aeaa63762224] rejected MsgVote from 3ba9ae53cc4335db [logterm: 2, index: 1261] at term 5"} {"level":"warn","ts":"2026-05-09T13:14:32.529031Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"227.003078ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:3 size:389"} {"level":"info","ts":"2026-05-09T13:14:32.529119Z","caller":"traceutil/trace.go:171","msg":"trace[2123556336] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:3; response_revision:1100; }","duration":"227.13639ms","start":"2026-05-09T13:14:32.301962Z","end":"2026-05-09T13:14:32.529098Z","steps":["trace[2123556336] 'agreement among raft nodes before linearized reading' (duration: 97.070631ms)","trace[2123556336] 'range keys from in-memory index tree' (duration: 129.904807ms)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:32.529160Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"129.995857ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-05-09T13:14:32.535877Z","caller":"traceutil/trace.go:171","msg":"trace[875334605] linearizableReadLoop","detail":"{readStateIndex:1277; appliedIndex:1278; }","duration":"136.867175ms","start":"2026-05-09T13:14:32.398982Z","end":"2026-05-09T13:14:32.535850Z","steps":["trace[875334605] 'read index received' (duration: 136.857515ms)","trace[875334605] 'applied index is now lower than readState.Index' (duration: 7.73µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:32.536143Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"171.702984ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-05-09T13:14:32.536196Z","caller":"traceutil/trace.go:171","msg":"trace[1594933397] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1101; }","duration":"171.804835ms","start":"2026-05-09T13:14:32.364378Z","end":"2026-05-09T13:14:32.536183Z","steps":["trace[1594933397] 'agreement among raft nodes before linearized reading' (duration: 171.649943ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:32.536472Z","caller":"traceutil/trace.go:171","msg":"trace[4054356] transaction","detail":"{read_only:false; response_revision:1102; number_of_response:1; }","duration":"233.644363ms","start":"2026-05-09T13:14:32.302748Z","end":"2026-05-09T13:14:32.536392Z","steps":["trace[4054356] 'process raft request' (duration: 233.297549ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:32.537123Z","caller":"traceutil/trace.go:171","msg":"trace[788618919] transaction","detail":"{read_only:false; response_revision:1106; number_of_response:1; }","duration":"233.673924ms","start":"2026-05-09T13:14:32.303421Z","end":"2026-05-09T13:14:32.537095Z","steps":["trace[788618919] 'process raft request' (duration: 233.005335ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:32.537235Z","caller":"traceutil/trace.go:171","msg":"trace[137548365] transaction","detail":"{read_only:false; response_revision:1107; number_of_response:1; }","duration":"219.233448ms","start":"2026-05-09T13:14:32.317990Z","end":"2026-05-09T13:14:32.537223Z","steps":["trace[137548365] 'process raft request' (duration: 219.170646ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:32.817097Z","caller":"traceutil/trace.go:171","msg":"trace[1104155810] transaction","detail":"{read_only:false; response_revision:1128; number_of_response:1; }","duration":"160.437889ms","start":"2026-05-09T13:14:32.656642Z","end":"2026-05-09T13:14:32.817080Z","steps":["trace[1104155810] 'process raft request' (duration: 160.318798ms)"],"step_count":1} {"level":"info","ts":"2026-05-09T13:14:32.817471Z","caller":"traceutil/trace.go:171","msg":"trace[1920884577] transaction","detail":"{read_only:false; response_revision:1130; number_of_response:1; }","duration":"128.265955ms","start":"2026-05-09T13:14:32.689194Z","end":"2026-05-09T13:14:32.817460Z","steps":["trace[1920884577] 'process raft request' (duration: 128.132712ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:33.222660Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"faf479c52908c230","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"102.110032ms"} {"level":"warn","ts":"2026-05-09T13:14:33.222744Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3ba9ae53cc4335db","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"102.211464ms"} {"level":"warn","ts":"2026-05-09T13:14:35.134179Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"faf479c52908c230","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"337.115705ms"} {"level":"warn","ts":"2026-05-09T13:14:35.134226Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3ba9ae53cc4335db","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"337.173345ms"} {"level":"warn","ts":"2026-05-09T13:14:35.134371Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"590.297139ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-05-09T13:14:35.134402Z","caller":"traceutil/trace.go:171","msg":"trace[573550342] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1183; }","duration":"590.377259ms","start":"2026-05-09T13:14:34.544013Z","end":"2026-05-09T13:14:35.134391Z","steps":["trace[573550342] 'range keys from in-memory index tree' (duration: 589.154635ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:35.134428Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:34.543995Z","time spent":"590.427401ms","remote":"127.0.0.1:34672","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-05-09T13:14:35.144980Z","caller":"traceutil/trace.go:171","msg":"trace[2051063083] linearizableReadLoop","detail":"{readStateIndex:1364; appliedIndex:1364; }","duration":"364.842299ms","start":"2026-05-09T13:14:34.780113Z","end":"2026-05-09T13:14:35.144956Z","steps":["trace[2051063083] 'read index received' (duration: 364.837149ms)","trace[2051063083] 'applied index is now lower than readState.Index' (duration: 3.58µs)"],"step_count":2} {"level":"warn","ts":"2026-05-09T13:14:35.145117Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"364.988191ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-05-09T13:14:35.145146Z","caller":"traceutil/trace.go:171","msg":"trace[1591725872] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1184; }","duration":"365.063312ms","start":"2026-05-09T13:14:34.780077Z","end":"2026-05-09T13:14:35.145140Z","steps":["trace[1591725872] 'agreement among raft nodes before linearized reading' (duration: 364.96456ms)"],"step_count":1} {"level":"warn","ts":"2026-05-09T13:14:35.145171Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-05-09T13:14:34.780058Z","time spent":"365.108492ms","remote":"127.0.0.1:34420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "}