{"level":"warn","ts":"2026-03-17T20:48:20.467542Z","caller":"embed/config.go:689","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-17T20:48:20.467877Z","caller":"etcdmain/etcd.go:73","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.19.213.83:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--experimental-initial-corrupt-check=true","--experimental-watch-progress-notify-interval=5s","--initial-advertise-peer-urls=https://199.19.213.83:2380","--initial-cluster=controller-1=https://199.19.213.3:2380,controller-3=https://199.19.213.83:2380,controller-2=https://199.19.213.102:2380","--initial-cluster-state=existing","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.19.213.83:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.19.213.83:2380","--name=controller-3","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt"]} {"level":"warn","ts":"2026-03-17T20:48:20.468040Z","caller":"embed/config.go:689","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-17T20:48:20.468060Z","caller":"embed/etcd.go:140","msg":"configuring peer listeners","listen-peer-urls":["https://199.19.213.83:2380"]} {"level":"info","ts":"2026-03-17T20:48:20.468244Z","caller":"embed/etcd.go:528","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-17T20:48:20.469644Z","caller":"embed/etcd.go:148","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.83:2379"]} {"level":"info","ts":"2026-03-17T20:48:20.471804Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.5.21","git-sha":"a17edfd","go-version":"go1.23.7","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-3","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.19.213.83:2380"],"listen-peer-urls":["https://199.19.213.83:2380"],"advertise-client-urls":["https://199.19.213.83:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.83:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://199.19.213.3:2380,controller-2=https://199.19.213.102:2380,controller-3=https://199.19.213.83:2380","initial-cluster-state":"existing","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"} {"level":"info","ts":"2026-03-17T20:48:20.477229Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"5.028295ms"} {"level":"info","ts":"2026-03-17T20:48:20.489832Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://199.19.213.3:2380","host":"199.19.213.3:2380","resolved-addr":"199.19.213.3:2380"} {"level":"info","ts":"2026-03-17T20:48:20.489890Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://199.19.213.102:2380","host":"199.19.213.102:2380","resolved-addr":"199.19.213.102:2380"} {"level":"info","ts":"2026-03-17T20:48:20.489914Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://199.19.213.83:2380","host":"199.19.213.83:2380","resolved-addr":"199.19.213.83:2380"} {"level":"info","ts":"2026-03-17T20:48:20.489965Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://199.19.213.102:2380","host":"199.19.213.102:2380","resolved-addr":"199.19.213.102:2380"} {"level":"info","ts":"2026-03-17T20:48:20.489979Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://199.19.213.83:2380","host":"199.19.213.83:2380","resolved-addr":"199.19.213.83:2380"} {"level":"info","ts":"2026-03-17T20:48:20.489988Z","caller":"netutil/netutil.go:112","msg":"resolved URL Host","url":"https://199.19.213.3:2380","host":"199.19.213.3:2380","resolved-addr":"199.19.213.3:2380"} {"level":"info","ts":"2026-03-17T20:48:20.510475Z","caller":"etcdserver/raft.go:506","msg":"starting local member","local-member-id":"5a110d012eba26a2","cluster-id":"7b1ae51097bedd0e"} {"level":"info","ts":"2026-03-17T20:48:20.510591Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 switched to configuration voters=()"} {"level":"info","ts":"2026-03-17T20:48:20.510627Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became follower at term 0"} {"level":"info","ts":"2026-03-17T20:48:20.510644Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"newRaft 5a110d012eba26a2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"warn","ts":"2026-03-17T20:48:20.525865Z","caller":"auth/store.go:1241","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-17T20:48:20.529628Z","caller":"mvcc/kvstore.go:425","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-17T20:48:20.529675Z","caller":"etcdserver/server.go:628","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-03-17T20:48:20.532699Z","caller":"etcdserver/quota.go:94","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-17T20:48:20.536334Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.536358Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333","remote-peer-urls":["https://199.19.213.3:2380"]} {"level":"info","ts":"2026-03-17T20:48:20.536380Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.536388Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf","remote-peer-urls":["https://199.19.213.102:2380"]} {"level":"info","ts":"2026-03-17T20:48:20.536404Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.536419Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.539582Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.540033Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.541074Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.541538Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333","remote-peer-urls":["https://199.19.213.3:2380"]} {"level":"info","ts":"2026-03-17T20:48:20.541862Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.542006Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.542353Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.542590Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf","remote-peer-urls":["https://199.19.213.102:2380"]} {"level":"info","ts":"2026-03-17T20:48:20.542768Z","caller":"etcdserver/server.go:875","msg":"starting etcd server","local-member-id":"5a110d012eba26a2","local-server-version":"3.5.21","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-17T20:48:20.543873Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-17T20:48:20.545788Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.546155Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.546361Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.546395Z","caller":"rafthttp/stream.go:169","msg":"started stream writer with remote peer","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.546422Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.546529Z","caller":"rafthttp/stream.go:395","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.546816Z","caller":"etcdserver/server.go:775","msg":"starting initial election tick advance","election-ticks":10} {"level":"info","ts":"2026-03-17T20:48:20.546990Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-17T20:48:20.547113Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-17T20:48:20.547191Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-17T20:48:20.548283Z","caller":"embed/etcd.go:762","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-17T20:48:20.548584Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"5a110d012eba26a2","initial-advertise-peer-urls":["https://199.19.213.83:2380"],"listen-peer-urls":["https://199.19.213.83:2380"],"advertise-client-urls":["https://199.19.213.83:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.83:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-17T20:48:20.548645Z","caller":"embed/etcd.go:908","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-17T20:48:20.548845Z","caller":"embed/etcd.go:633","msg":"serving peer traffic","address":"199.19.213.83:2380"} {"level":"info","ts":"2026-03-17T20:48:20.548879Z","caller":"embed/etcd.go:603","msg":"cmux::serve","address":"199.19.213.83:2380"} {"level":"info","ts":"2026-03-17T20:48:20.549146Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [term: 0] received a MsgHeartbeat message with higher term from 3cf904ec39d3e333 [term: 2]"} {"level":"info","ts":"2026-03-17T20:48:20.549177Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became follower at term 2"} {"level":"info","ts":"2026-03-17T20:48:20.549191Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: 5a110d012eba26a2 elected leader 3cf904ec39d3e333 at term 2"} {"level":"info","ts":"2026-03-17T20:48:20.555811Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"5a110d012eba26a2","to":"3cf904ec39d3e333","stream-type":"stream Message"} {"level":"info","ts":"2026-03-17T20:48:20.555823Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"5a110d012eba26a2","to":"77297b5b7846c4bf","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-17T20:48:20.555875Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.555893Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.555847Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.555916Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.556841Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"5a110d012eba26a2","to":"77297b5b7846c4bf","stream-type":"stream Message"} {"level":"info","ts":"2026-03-17T20:48:20.556881Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.557277Z","caller":"rafthttp/stream.go:249","msg":"set message encoder","from":"5a110d012eba26a2","to":"3cf904ec39d3e333","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-17T20:48:20.560588Z","caller":"rafthttp/stream.go:274","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.559354Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.560849Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.562655Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5a110d012eba26a2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-17T20:48:20.563006Z","caller":"rafthttp/stream.go:412","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"5a110d012eba26a2","remote-peer-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.594652Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 switched to configuration voters=(4393548324105610035)"} {"level":"info","ts":"2026-03-17T20:48:20.594789Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"7b1ae51097bedd0e","local-member-id":"5a110d012eba26a2","added-peer-id":"3cf904ec39d3e333","added-peer-peer-urls":["https://199.19.213.3:2380"],"added-peer-is-learner":false} {"level":"info","ts":"2026-03-17T20:48:20.594932Z","caller":"membership/cluster.go:587","msg":"set initial cluster version","cluster-id":"7b1ae51097bedd0e","local-member-id":"5a110d012eba26a2","cluster-version":"3.5"} {"level":"info","ts":"2026-03-17T20:48:20.595006Z","caller":"api/capability.go:75","msg":"enabled capabilities for version","cluster-version":"3.5"} {"level":"info","ts":"2026-03-17T20:48:20.597785Z","caller":"etcdserver/server.go:798","msg":"initialized peer connections; fast-forwarding election ticks","local-member-id":"5a110d012eba26a2","forward-ticks":8,"forward-duration":"800ms","election-ticks":10,"election-timeout":"1s","active-remote-members":2} {"level":"info","ts":"2026-03-17T20:48:20.640943Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 switched to configuration voters=(4393548324105610035) learners=(8586529797348705471)"} {"level":"info","ts":"2026-03-17T20:48:20.641046Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"7b1ae51097bedd0e","local-member-id":"5a110d012eba26a2","added-peer-id":"77297b5b7846c4bf","added-peer-peer-urls":["https://199.19.213.102:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-17T20:48:20.641421Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 switched to configuration voters=(4393548324105610035 8586529797348705471)"} {"level":"info","ts":"2026-03-17T20:48:20.641477Z","caller":"membership/cluster.go:537","msg":"promote member","cluster-id":"7b1ae51097bedd0e","local-member-id":"5a110d012eba26a2","promoted-member-id":"77297b5b7846c4bf"} {"level":"info","ts":"2026-03-17T20:48:20.687725Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 switched to configuration voters=(4393548324105610035 8586529797348705471) learners=(6489982836747675298)"} {"level":"info","ts":"2026-03-17T20:48:20.687856Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"7b1ae51097bedd0e","local-member-id":"5a110d012eba26a2","added-peer-id":"5a110d012eba26a2","added-peer-peer-urls":["https://199.19.213.83:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-17T20:48:20.694409Z","caller":"etcdserver/server.go:2144","msg":"published local member to cluster through raft","local-member-id":"5a110d012eba26a2","local-member-attributes":"{Name:controller-3 ClientURLs:[https://199.19.213.83:2379]}","request-path":"/0/members/5a110d012eba26a2/attributes","cluster-id":"7b1ae51097bedd0e","publish-timeout":"7s"} {"level":"info","ts":"2026-03-17T20:48:20.694459Z","caller":"embed/serve.go:124","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-17T20:48:20.694764Z","caller":"embed/serve.go:124","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-17T20:48:20.694859Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-17T20:48:20.694928Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-03-17T20:48:20.696094Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-17T20:48:20.697398Z","caller":"embed/serve.go:275","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.19.213.83:2379"} {"level":"info","ts":"2026-03-17T20:48:20.698412Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-17T20:48:20.699189Z","caller":"embed/serve.go:275","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"warn","ts":"2026-03-17T20:48:20.703673Z","caller":"embed/config_logging.go:170","msg":"rejected connection on client endpoint","remote-addr":"199.19.213.83:49268","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-17T20:48:21.084275Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 switched to configuration voters=(4393548324105610035 6489982836747675298 8586529797348705471)"} {"level":"info","ts":"2026-03-17T20:48:21.084413Z","caller":"membership/cluster.go:537","msg":"promote member","cluster-id":"7b1ae51097bedd0e","local-member-id":"5a110d012eba26a2","promoted-member-id":"5a110d012eba26a2"} {"level":"info","ts":"2026-03-17T20:48:40.994211Z","caller":"traceutil/trace.go:171","msg":"trace[1186088789] linearizableReadLoop","detail":"{readStateIndex:780; appliedIndex:780; }","duration":"386.076141ms","start":"2026-03-17T20:48:40.608120Z","end":"2026-03-17T20:48:40.994196Z","steps":["trace[1186088789] 'read index received' (duration: 386.072291ms)","trace[1186088789] 'applied index is now lower than readState.Index' (duration: 3.09µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:48:40.994534Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"386.384267ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:508"} {"level":"info","ts":"2026-03-17T20:48:40.994601Z","caller":"traceutil/trace.go:171","msg":"trace[262980764] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:708; }","duration":"386.504669ms","start":"2026-03-17T20:48:40.608079Z","end":"2026-03-17T20:48:40.994584Z","steps":["trace[262980764] 'agreement among raft nodes before linearized reading' (duration: 386.344876ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:40.994634Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:40.608061Z","time spent":"386.56622ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":530,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:48:40.995359Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"238.217395ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:48:40.995404Z","caller":"traceutil/trace.go:171","msg":"trace[1484926678] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:708; }","duration":"238.268786ms","start":"2026-03-17T20:48:40.757122Z","end":"2026-03-17T20:48:40.995391Z","steps":["trace[1484926678] 'agreement among raft nodes before linearized reading' (duration: 238.203355ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:46.048629Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808177,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:48:46.504936Z","caller":"traceutil/trace.go:171","msg":"trace[1685570209] linearizableReadLoop","detail":"{readStateIndex:800; appliedIndex:800; }","duration":"956.706216ms","start":"2026-03-17T20:48:45.548198Z","end":"2026-03-17T20:48:46.504904Z","steps":["trace[1685570209] 'read index received' (duration: 956.694785ms)","trace[1685570209] 'applied index is now lower than readState.Index' (duration: 9.861µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:48:46.506275Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"958.03068ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-17T20:48:46.506423Z","caller":"traceutil/trace.go:171","msg":"trace[887850894] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:725; }","duration":"958.231243ms","start":"2026-03-17T20:48:45.548167Z","end":"2026-03-17T20:48:46.506398Z","steps":["trace[887850894] 'agreement among raft nodes before linearized reading' (duration: 956.843227ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:46.506490Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:45.548153Z","time spent":"958.313236ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":504,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:48:46.507350Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"750.918392ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:48:46.507492Z","caller":"traceutil/trace.go:171","msg":"trace[1539149509] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:726; }","duration":"751.060265ms","start":"2026-03-17T20:48:45.756417Z","end":"2026-03-17T20:48:46.507477Z","steps":["trace[1539149509] 'agreement among raft nodes before linearized reading' (duration: 750.894582ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:46.507424Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"519.919659ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:48:46.507597Z","caller":"traceutil/trace.go:171","msg":"trace[1495656350] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:726; }","duration":"520.171773ms","start":"2026-03-17T20:48:45.987404Z","end":"2026-03-17T20:48:46.507576Z","steps":["trace[1495656350] 'agreement among raft nodes before linearized reading' (duration: 519.911089ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:46.507795Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:45.987387Z","time spent":"520.392078ms","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:48:49.560918Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808190,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:48:51.009619Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808190,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:48:51.779911Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808190,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:48:52.368760Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808190,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:48:52.410003Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"3.409175263s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-17T20:48:52.869648Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808190,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:48:53.016283Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"647.118634ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:48:53.016349Z","caller":"traceutil/trace.go:171","msg":"trace[116433818] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"647.221318ms","start":"2026-03-17T20:48:52.369113Z","end":"2026-03-17T20:48:53.016335Z","steps":["trace[116433818] 'agreement among raft nodes before linearized reading' (duration: 647.137755ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:53.016393Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:52.369101Z","time spent":"647.283708ms","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:48:53 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-17T20:48:53.017903Z","caller":"traceutil/trace.go:171","msg":"trace[750704142] linearizableReadLoop","detail":"{readStateIndex:811; appliedIndex:810; }","duration":"3.957747283s","start":"2026-03-17T20:48:49.060142Z","end":"2026-03-17T20:48:53.017889Z","steps":["trace[750704142] 'read index received' (duration: 3.946087932s)","trace[750704142] 'applied index is now lower than readState.Index' (duration: 11.658201ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:48:53.017990Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.957840515s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-17T20:48:53.018014Z","caller":"traceutil/trace.go:171","msg":"trace[371893251] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:739; }","duration":"3.957888595s","start":"2026-03-17T20:48:49.060118Z","end":"2026-03-17T20:48:53.018007Z","steps":["trace[371893251] 'agreement among raft nodes before linearized reading' (duration: 3.957839164s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:53.018083Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:49.060108Z","time spent":"3.957966327s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":504,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:48:53.019183Z","caller":"etcdserver/v3_server.go:897","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":2783960132499808193,"received-request-id":2783960132499808190} {"level":"info","ts":"2026-03-17T20:48:53.020455Z","caller":"traceutil/trace.go:171","msg":"trace[1624021480] transaction","detail":"{read_only:false; response_revision:742; number_of_response:1; }","duration":"388.280829ms","start":"2026-03-17T20:48:52.632153Z","end":"2026-03-17T20:48:53.020433Z","steps":["trace[1624021480] 'process raft request' (duration: 388.251448ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:48:53.020473Z","caller":"traceutil/trace.go:171","msg":"trace[639705808] transaction","detail":"{read_only:false; response_revision:741; number_of_response:1; }","duration":"574.336101ms","start":"2026-03-17T20:48:52.446120Z","end":"2026-03-17T20:48:53.020456Z","steps":["trace[639705808] 'process raft request' (duration: 574.226799ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:53.020937Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:52.632136Z","time spent":"388.365851ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:48:53.020954Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:52.446100Z","time spent":"574.430232ms","remote":"127.0.0.1:59830","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4049,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:48:53.022487Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.623224447s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-17T20:48:53.022576Z","caller":"traceutil/trace.go:171","msg":"trace[1676998118] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:742; }","duration":"3.623317288s","start":"2026-03-17T20:48:49.399210Z","end":"2026-03-17T20:48:53.022527Z","steps":["trace[1676998118] 'agreement among raft nodes before linearized reading' (duration: 3.623221826s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:53.022564Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"605.906585ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:48:53.022749Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:49.399188Z","time spent":"3.62342096s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":462,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:48:53.022501Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"604.210164ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"warn","ts":"2026-03-17T20:48:53.022649Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"609.956509ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-17T20:48:53.022846Z","caller":"traceutil/trace.go:171","msg":"trace[1148070679] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:742; }","duration":"604.578721ms","start":"2026-03-17T20:48:52.418254Z","end":"2026-03-17T20:48:53.022832Z","steps":["trace[1148070679] 'agreement among raft nodes before linearized reading' (duration: 604.197265ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:48:53.022853Z","caller":"traceutil/trace.go:171","msg":"trace[800728860] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:742; }","duration":"610.197574ms","start":"2026-03-17T20:48:52.412646Z","end":"2026-03-17T20:48:53.022844Z","steps":["trace[800728860] 'agreement among raft nodes before linearized reading' (duration: 609.962119ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:53.022880Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:52.418238Z","time spent":"604.632032ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":459,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-17T20:48:53.022755Z","caller":"traceutil/trace.go:171","msg":"trace[112757519] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:742; }","duration":"606.095169ms","start":"2026-03-17T20:48:52.416645Z","end":"2026-03-17T20:48:53.022741Z","steps":["trace[112757519] 'agreement among raft nodes before linearized reading' (duration: 605.881825ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:53.023041Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:48:52.412617Z","time spent":"610.258784ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":531,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"info","ts":"2026-03-17T20:48:56.924630Z","caller":"traceutil/trace.go:171","msg":"trace[1788916349] linearizableReadLoop","detail":"{readStateIndex:837; appliedIndex:837; }","duration":"143.91419ms","start":"2026-03-17T20:48:56.780686Z","end":"2026-03-17T20:48:56.924600Z","steps":["trace[1788916349] 'read index received' (duration: 143.90861ms)","trace[1788916349] 'applied index is now lower than readState.Index' (duration: 4.27µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:48:56.925116Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"144.504401ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:48:56.925169Z","caller":"traceutil/trace.go:171","msg":"trace[1832881769] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:755; }","duration":"144.582702ms","start":"2026-03-17T20:48:56.780574Z","end":"2026-03-17T20:48:56.925156Z","steps":["trace[1832881769] 'agreement among raft nodes before linearized reading' (duration: 144.147415ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:56.926139Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"143.99895ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:508"} {"level":"info","ts":"2026-03-17T20:48:56.926192Z","caller":"traceutil/trace.go:171","msg":"trace[1358430249] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:755; }","duration":"144.103993ms","start":"2026-03-17T20:48:56.782076Z","end":"2026-03-17T20:48:56.926180Z","steps":["trace[1358430249] 'agreement among raft nodes before linearized reading' (duration: 143.996321ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:48:59.388834Z","caller":"traceutil/trace.go:171","msg":"trace[1567379601] linearizableReadLoop","detail":"{readStateIndex:873; appliedIndex:873; }","duration":"224.558692ms","start":"2026-03-17T20:48:59.164251Z","end":"2026-03-17T20:48:59.388810Z","steps":["trace[1567379601] 'read index received' (duration: 224.553682ms)","trace[1567379601] 'applied index is now lower than readState.Index' (duration: 3.85µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:48:59.389144Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"224.875407ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:436"} {"level":"info","ts":"2026-03-17T20:48:59.389201Z","caller":"traceutil/trace.go:171","msg":"trace[878601824] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:787; }","duration":"224.98398ms","start":"2026-03-17T20:48:59.164207Z","end":"2026-03-17T20:48:59.389191Z","steps":["trace[878601824] 'agreement among raft nodes before linearized reading' (duration: 224.854567ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:48:59.445020Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"155.527243ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-17T20:48:59.445111Z","caller":"traceutil/trace.go:171","msg":"trace[1275015865] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:789; }","duration":"155.657025ms","start":"2026-03-17T20:48:59.289434Z","end":"2026-03-17T20:48:59.445091Z","steps":["trace[1275015865] 'agreement among raft nodes before linearized reading' (duration: 147.204592ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:04.497643Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:04.062303Z","time spent":"435.337819ms","remote":"127.0.0.1:59690","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-17T20:49:08.373426Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"273.188502ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"info","ts":"2026-03-17T20:49:08.373573Z","caller":"traceutil/trace.go:171","msg":"trace[1211562582] linearizableReadLoop","detail":"{readStateIndex:906; appliedIndex:905; }","duration":"331.336988ms","start":"2026-03-17T20:49:08.042215Z","end":"2026-03-17T20:49:08.373552Z","steps":["trace[1211562582] 'read index received' (duration: 58.508272ms)","trace[1211562582] 'applied index is now lower than readState.Index' (duration: 272.826996ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:08.373694Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"331.474322ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-17T20:49:08.373724Z","caller":"traceutil/trace.go:171","msg":"trace[1693917350] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:816; }","duration":"331.551382ms","start":"2026-03-17T20:49:08.042164Z","end":"2026-03-17T20:49:08.373716Z","steps":["trace[1693917350] 'agreement among raft nodes before linearized reading' (duration: 331.44026ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:08.373757Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:08.042146Z","time spent":"331.602914ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":462,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:09.549519Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"132.080464ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-17T20:49:09.549889Z","caller":"traceutil/trace.go:171","msg":"trace[1594674640] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:817; }","duration":"132.510602ms","start":"2026-03-17T20:49:09.417358Z","end":"2026-03-17T20:49:09.549869Z","steps":["trace[1594674640] 'range keys from in-memory index tree' (duration: 130.422923ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:10.154160Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"144.21917ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"warn","ts":"2026-03-17T20:49:10.154620Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"397.985022ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:10.154694Z","caller":"traceutil/trace.go:171","msg":"trace[884580143] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:819; }","duration":"398.068483ms","start":"2026-03-17T20:49:09.756608Z","end":"2026-03-17T20:49:10.154676Z","steps":["trace[884580143] 'agreement among raft nodes before linearized reading' (duration: 397.960501ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:10.154467Z","caller":"traceutil/trace.go:171","msg":"trace[1208029322] linearizableReadLoop","detail":"{readStateIndex:912; appliedIndex:910; }","duration":"397.831399ms","start":"2026-03-17T20:49:09.756615Z","end":"2026-03-17T20:49:10.154446Z","steps":["trace[1208029322] 'read index received' (duration: 191.544504ms)","trace[1208029322] 'applied index is now lower than readState.Index' (duration: 206.284235ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:10.230798Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"271.039189ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"warn","ts":"2026-03-17T20:49:10.230796Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"238.002496ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:10.230894Z","caller":"traceutil/trace.go:171","msg":"trace[354370402] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:821; }","duration":"271.186431ms","start":"2026-03-17T20:49:09.959684Z","end":"2026-03-17T20:49:10.230870Z","steps":["trace[354370402] 'agreement among raft nodes before linearized reading' (duration: 271.034569ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:10.230900Z","caller":"traceutil/trace.go:171","msg":"trace[1117537285] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:821; }","duration":"238.219249ms","start":"2026-03-17T20:49:09.992663Z","end":"2026-03-17T20:49:10.230882Z","steps":["trace[1117537285] 'agreement among raft nodes before linearized reading' (duration: 238.079386ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:18.802228Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808311,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:18.896799Z","caller":"traceutil/trace.go:171","msg":"trace[245363156] linearizableReadLoop","detail":"{readStateIndex:987; appliedIndex:987; }","duration":"595.690764ms","start":"2026-03-17T20:49:18.301093Z","end":"2026-03-17T20:49:18.896784Z","steps":["trace[245363156] 'read index received' (duration: 595.686704ms)","trace[245363156] 'applied index is now lower than readState.Index' (duration: 3.33µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:18.896965Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"595.851166ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-17T20:49:18.897070Z","caller":"traceutil/trace.go:171","msg":"trace[768638962] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:890; }","duration":"595.984548ms","start":"2026-03-17T20:49:18.301065Z","end":"2026-03-17T20:49:18.897050Z","steps":["trace[768638962] 'agreement among raft nodes before linearized reading' (duration: 595.773765ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:18.897247Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:18.301025Z","time spent":"596.128171ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":459,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:18.898646Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"252.007727ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:439"} {"level":"info","ts":"2026-03-17T20:49:18.898680Z","caller":"traceutil/trace.go:171","msg":"trace[313398756] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:890; }","duration":"252.078199ms","start":"2026-03-17T20:49:18.646594Z","end":"2026-03-17T20:49:18.898672Z","steps":["trace[313398756] 'agreement among raft nodes before linearized reading' (duration: 252.016598ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:18.898802Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"141.727375ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:18.898916Z","caller":"traceutil/trace.go:171","msg":"trace[1721732756] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:890; }","duration":"141.848217ms","start":"2026-03-17T20:49:18.757035Z","end":"2026-03-17T20:49:18.898883Z","steps":["trace[1721732756] 'agreement among raft nodes before linearized reading' (duration: 141.679454ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:20.352026Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:20.852863Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:21.353035Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:21.853447Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:21.989247Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000654673s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:49:21.989423Z","caller":"traceutil/trace.go:171","msg":"trace[139367955] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000879177s","start":"2026-03-17T20:49:19.988521Z","end":"2026-03-17T20:49:21.989400Z","steps":["trace[139367955] 'agreement among raft nodes before linearized reading' (duration: 2.000685824s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:21.989487Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:19.988499Z","time spent":"2.000973348s","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:49:21.993884Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"2.921992362s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-17T20:49:22.354218Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:22.855074Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:23.356180Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:23.856713Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:23.989749Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.999877523s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:49:23.989832Z","caller":"traceutil/trace.go:171","msg":"trace[253162887] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000136427s","start":"2026-03-17T20:49:21.989676Z","end":"2026-03-17T20:49:23.989813Z","steps":["trace[253162887] 'agreement among raft nodes before linearized reading' (duration: 1.999904453s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:23.989881Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:21.989659Z","time spent":"2.000213139s","remote":"127.0.0.1:59678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:49:23 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:24.357568Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:24.851981Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.000355998s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:49:24.852059Z","caller":"traceutil/trace.go:171","msg":"trace[1223249923] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"5.00049187s","start":"2026-03-17T20:49:19.851550Z","end":"2026-03-17T20:49:24.852042Z","steps":["trace[1223249923] 'agreement among raft nodes before linearized reading' (duration: 5.000384598s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:24.852111Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:19.851532Z","time spent":"5.000566202s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} 2026/03/17 20:49:24 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:24.858094Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:25.358191Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:25.449416Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.99967459s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:49:25.449497Z","caller":"traceutil/trace.go:171","msg":"trace[1360941768] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999807743s","start":"2026-03-17T20:49:20.449670Z","end":"2026-03-17T20:49:25.449478Z","steps":["trace[1360941768] 'agreement among raft nodes before linearized reading' (duration: 4.999700381s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:25.449583Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:20.449653Z","time spent":"4.999919025s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} 2026/03/17 20:49:25 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:25.657871Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"2.325086787s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-17T20:49:25.664509Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.907480913s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:25.664580Z","caller":"traceutil/trace.go:171","msg":"trace[1405000423] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:890; }","duration":"5.907565424s","start":"2026-03-17T20:49:19.757000Z","end":"2026-03-17T20:49:25.664566Z","steps":["trace[1405000423] 'range keys from in-memory index tree' (duration: 5.907442822s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:25.665486Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.427412766s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:25.666517Z","caller":"traceutil/trace.go:171","msg":"trace[374040481] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:890; }","duration":"3.428442575s","start":"2026-03-17T20:49:22.238061Z","end":"2026-03-17T20:49:25.666504Z","steps":["trace[374040481] 'range keys from in-memory index tree' (duration: 3.427367625s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:25.859013Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808313,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:25.926846Z","caller":"traceutil/trace.go:171","msg":"trace[1407483481] linearizableReadLoop","detail":"{readStateIndex:992; appliedIndex:988; }","duration":"6.075232258s","start":"2026-03-17T20:49:19.851588Z","end":"2026-03-17T20:49:25.926820Z","steps":["trace[1407483481] 'read index received' (duration: 6.07483821s)","trace[1407483481] 'applied index is now lower than readState.Index' (duration: 393.258µs)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:49:25.943146Z","caller":"traceutil/trace.go:171","msg":"trace[1122285115] transaction","detail":"{read_only:false; response_revision:899; number_of_response:1; }","duration":"1.805063317s","start":"2026-03-17T20:49:24.138002Z","end":"2026-03-17T20:49:25.943065Z","steps":["trace[1122285115] 'process raft request' (duration: 1.804863374s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:25.943285Z","caller":"traceutil/trace.go:171","msg":"trace[1499806446] transaction","detail":"{read_only:false; response_revision:898; number_of_response:1; }","duration":"2.490586454s","start":"2026-03-17T20:49:23.452659Z","end":"2026-03-17T20:49:25.943246Z","steps":["trace[1499806446] 'process raft request' (duration: 2.489929212s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:25.943588Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:23.452636Z","time spent":"2.49084704s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:49:25.943673Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:24.137882Z","time spent":"1.805351363s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":676,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:49:25.956548Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.635775201s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:439"} {"level":"warn","ts":"2026-03-17T20:49:25.956619Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.958709917s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:49:25.956626Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.78549286s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"warn","ts":"2026-03-17T20:49:25.956648Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"360.840481ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:49:25.956558Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.894816248s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.19.213.83\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-17T20:49:25.956667Z","caller":"traceutil/trace.go:171","msg":"trace[374165695] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:901; }","duration":"360.898442ms","start":"2026-03-17T20:49:25.595761Z","end":"2026-03-17T20:49:25.956659Z","steps":["trace[374165695] 'agreement among raft nodes before linearized reading' (duration: 360.877232ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:25.956663Z","caller":"traceutil/trace.go:171","msg":"trace[908876403] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:901; }","duration":"3.785575061s","start":"2026-03-17T20:49:22.171077Z","end":"2026-03-17T20:49:25.956652Z","steps":["trace[908876403] 'agreement among raft nodes before linearized reading' (duration: 3.7855012s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:25.956676Z","caller":"traceutil/trace.go:171","msg":"trace[1316066226] range","detail":"{range_begin:/registry/masterleases/199.19.213.83; range_end:; response_count:1; response_revision:901; }","duration":"1.894983581s","start":"2026-03-17T20:49:24.061681Z","end":"2026-03-17T20:49:25.956665Z","steps":["trace[1316066226] 'agreement among raft nodes before linearized reading' (duration: 1.894806358s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:25.956689Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:25.595743Z","time spent":"360.942092ms","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:49:25.956697Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:22.171064Z","time spent":"3.785624793s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":459,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:25.956717Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:24.061666Z","time spent":"1.895038532s","remote":"127.0.0.1:59690","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":1,"response size":155,"request content":"key:\"/registry/masterleases/199.19.213.83\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:25.956576Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.963800238s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-17T20:49:25.956852Z","caller":"traceutil/trace.go:171","msg":"trace[2132984790] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4; range_end:; response_count:1; response_revision:901; }","duration":"1.964045103s","start":"2026-03-17T20:49:23.992760Z","end":"2026-03-17T20:49:25.956805Z","steps":["trace[2132984790] 'agreement among raft nodes before linearized reading' (duration: 1.963763158s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:25.956631Z","caller":"traceutil/trace.go:171","msg":"trace[604875271] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:901; }","duration":"4.635902843s","start":"2026-03-17T20:49:21.320709Z","end":"2026-03-17T20:49:25.956612Z","steps":["trace[604875271] 'agreement among raft nodes before linearized reading' (duration: 4.635762881s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:25.957038Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:21.320697Z","time spent":"4.63632751s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:25.956598Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"291.978014ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:25.957131Z","caller":"traceutil/trace.go:171","msg":"trace[1931744679] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:901; }","duration":"292.508223ms","start":"2026-03-17T20:49:25.664615Z","end":"2026-03-17T20:49:25.957124Z","steps":["trace[1931744679] 'agreement among raft nodes before linearized reading' (duration: 291.973184ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:25.956662Z","caller":"traceutil/trace.go:171","msg":"trace[898877679] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:901; }","duration":"1.958769058s","start":"2026-03-17T20:49:23.997875Z","end":"2026-03-17T20:49:25.956644Z","steps":["trace[898877679] 'agreement among raft nodes before linearized reading' (duration: 1.958726297s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:25.957287Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:23.997858Z","time spent":"1.95941532s","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:49:25.957003Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:23.992711Z","time spent":"1.964232496s","remote":"127.0.0.1:59748","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":816,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4\" limit:1 "} {"level":"info","ts":"2026-03-17T20:49:28.025162Z","caller":"traceutil/trace.go:171","msg":"trace[1214945779] linearizableReadLoop","detail":"{readStateIndex:1012; appliedIndex:1012; }","duration":"274.376815ms","start":"2026-03-17T20:49:27.750756Z","end":"2026-03-17T20:49:28.025133Z","steps":["trace[1214945779] 'read index received' (duration: 274.363285ms)","trace[1214945779] 'applied index is now lower than readState.Index' (duration: 11.88µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:28.025419Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"274.63851ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-17T20:49:28.025496Z","caller":"traceutil/trace.go:171","msg":"trace[543719698] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:909; }","duration":"274.761322ms","start":"2026-03-17T20:49:27.750718Z","end":"2026-03-17T20:49:28.025480Z","steps":["trace[543719698] 'agreement among raft nodes before linearized reading' (duration: 274.571939ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:28.026996Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"269.885685ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:28.027047Z","caller":"traceutil/trace.go:171","msg":"trace[2053176298] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:912; }","duration":"269.945126ms","start":"2026-03-17T20:49:27.757090Z","end":"2026-03-17T20:49:28.027035Z","steps":["trace[2053176298] 'agreement among raft nodes before linearized reading' (duration: 269.872165ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:33.027122Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808354,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:33.397412Z","caller":"traceutil/trace.go:171","msg":"trace[2095039068] linearizableReadLoop","detail":"{readStateIndex:1030; appliedIndex:1030; }","duration":"870.635704ms","start":"2026-03-17T20:49:32.526734Z","end":"2026-03-17T20:49:33.397370Z","steps":["trace[2095039068] 'read index received' (duration: 870.622974ms)","trace[2095039068] 'applied index is now lower than readState.Index' (duration: 11.19µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:33.397532Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"870.780207ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/\" range_end:\"/registry/pods0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-03-17T20:49:33.397569Z","caller":"traceutil/trace.go:171","msg":"trace[120433191] range","detail":"{range_begin:/registry/pods/; range_end:/registry/pods0; response_count:0; response_revision:924; }","duration":"870.871889ms","start":"2026-03-17T20:49:32.526687Z","end":"2026-03-17T20:49:33.397559Z","steps":["trace[120433191] 'agreement among raft nodes before linearized reading' (duration: 870.766497ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:33.397604Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:32.526673Z","time spent":"870.921469ms","remote":"127.0.0.1:59840","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":25,"response size":29,"request content":"key:\"/registry/pods/\" range_end:\"/registry/pods0\" count_only:true "} {"level":"info","ts":"2026-03-17T20:49:33.581023Z","caller":"traceutil/trace.go:171","msg":"trace[921856183] transaction","detail":"{read_only:false; response_revision:925; number_of_response:1; }","duration":"1.0544725s","start":"2026-03-17T20:49:32.526529Z","end":"2026-03-17T20:49:33.581001Z","steps":["trace[921856183] 'process raft request' (duration: 1.054222795s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:33.581207Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:32.526500Z","time spent":"1.054618493s","remote":"127.0.0.1:59840","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18315,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:49:33.585311Z","caller":"traceutil/trace.go:171","msg":"trace[1171803423] linearizableReadLoop","detail":"{readStateIndex:1033; appliedIndex:1033; }","duration":"187.816425ms","start":"2026-03-17T20:49:33.397472Z","end":"2026-03-17T20:49:33.585289Z","steps":["trace[1171803423] 'read index received' (duration: 187.810495ms)","trace[1171803423] 'applied index is now lower than readState.Index' (duration: 4.89µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:33.585456Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"307.118694ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:439"} {"level":"info","ts":"2026-03-17T20:49:33.585516Z","caller":"traceutil/trace.go:171","msg":"trace[1290527653] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:926; }","duration":"307.230575ms","start":"2026-03-17T20:49:33.278271Z","end":"2026-03-17T20:49:33.585501Z","steps":["trace[1290527653] 'agreement among raft nodes before linearized reading' (duration: 307.066062ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:33.585543Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"799.151323ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumeclaims/\" range_end:\"/registry/persistentvolumeclaims0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:49:33.585556Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:33.278251Z","time spent":"307.296457ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:33.585538Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"828.744338ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:33.585581Z","caller":"traceutil/trace.go:171","msg":"trace[1946580377] range","detail":"{range_begin:/registry/persistentvolumeclaims/; range_end:/registry/persistentvolumeclaims0; response_count:0; response_revision:926; }","duration":"799.239346ms","start":"2026-03-17T20:49:32.786331Z","end":"2026-03-17T20:49:33.585570Z","steps":["trace[1946580377] 'agreement among raft nodes before linearized reading' (duration: 799.139263ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:33.585622Z","caller":"traceutil/trace.go:171","msg":"trace[642793287] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:926; }","duration":"828.81793ms","start":"2026-03-17T20:49:32.756767Z","end":"2026-03-17T20:49:33.585585Z","steps":["trace[642793287] 'agreement among raft nodes before linearized reading' (duration: 828.725458ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:33.585628Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:32.786316Z","time spent":"799.298337ms","remote":"127.0.0.1:59816","response type":"/etcdserverpb.KV/Range","request count":0,"request size":72,"response count":0,"response size":27,"request content":"key:\"/registry/persistentvolumeclaims/\" range_end:\"/registry/persistentvolumeclaims0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:33.586589Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"977.417531ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-17T20:49:33.586736Z","caller":"traceutil/trace.go:171","msg":"trace[685167567] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:926; }","duration":"977.627634ms","start":"2026-03-17T20:49:32.609094Z","end":"2026-03-17T20:49:33.586721Z","steps":["trace[685167567] 'agreement among raft nodes before linearized reading' (duration: 977.41557ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:33.586834Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:32.609072Z","time spent":"977.747317ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":504,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"info","ts":"2026-03-17T20:49:34.058709Z","caller":"traceutil/trace.go:171","msg":"trace[1589268151] linearizableReadLoop","detail":"{readStateIndex:1034; appliedIndex:1034; }","duration":"473.302706ms","start":"2026-03-17T20:49:33.585383Z","end":"2026-03-17T20:49:34.058686Z","steps":["trace[1589268151] 'read index received' (duration: 473.296856ms)","trace[1589268151] 'applied index is now lower than readState.Index' (duration: 4.24µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:34.058889Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"560.996724ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:436"} {"level":"info","ts":"2026-03-17T20:49:34.059053Z","caller":"traceutil/trace.go:171","msg":"trace[266003374] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:926; }","duration":"561.229168ms","start":"2026-03-17T20:49:33.497817Z","end":"2026-03-17T20:49:34.059046Z","steps":["trace[266003374] 'agreement among raft nodes before linearized reading' (duration: 560.978403ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:34.059117Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:33.497801Z","time spent":"561.275119ms","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":458,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:34.110947Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"121.564427ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:34.111014Z","caller":"traceutil/trace.go:171","msg":"trace[612333201] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:934; }","duration":"121.68543ms","start":"2026-03-17T20:49:33.989314Z","end":"2026-03-17T20:49:34.111000Z","steps":["trace[612333201] 'agreement among raft nodes before linearized reading' (duration: 121.568428ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:34.111222Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"164.742316ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:508"} {"level":"info","ts":"2026-03-17T20:49:34.111267Z","caller":"traceutil/trace.go:171","msg":"trace[543671230] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:934; }","duration":"164.817266ms","start":"2026-03-17T20:49:33.946443Z","end":"2026-03-17T20:49:34.111260Z","steps":["trace[543671230] 'agreement among raft nodes before linearized reading' (duration: 164.749665ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:34.111587Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"355.073515ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:34.111659Z","caller":"traceutil/trace.go:171","msg":"trace[787688734] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:934; }","duration":"355.157596ms","start":"2026-03-17T20:49:33.756484Z","end":"2026-03-17T20:49:34.111641Z","steps":["trace[787688734] 'agreement among raft nodes before linearized reading' (duration: 354.958423ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:34.611831Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808359,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:34.871987Z","caller":"traceutil/trace.go:171","msg":"trace[1677078003] linearizableReadLoop","detail":"{readStateIndex:1042; appliedIndex:1042; }","duration":"761.113792ms","start":"2026-03-17T20:49:34.110851Z","end":"2026-03-17T20:49:34.871965Z","steps":["trace[1677078003] 'read index received' (duration: 761.106622ms)","trace[1677078003] 'applied index is now lower than readState.Index' (duration: 6.05µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:34.873552Z","caller":"etcdserver/v3_server.go:897","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":2783960132499808360,"received-request-id":2783960132499808359} {"level":"warn","ts":"2026-03-17T20:49:35.450833Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.388147909s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.19.213.83\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-17T20:49:35.450932Z","caller":"traceutil/trace.go:171","msg":"trace[1826938817] range","detail":"{range_begin:/registry/masterleases/199.19.213.83; range_end:; response_count:1; response_revision:934; }","duration":"1.38826231s","start":"2026-03-17T20:49:34.062639Z","end":"2026-03-17T20:49:35.450901Z","steps":["trace[1826938817] 'agreement among raft nodes before linearized reading' (duration: 809.425079ms)","trace[1826938817] 'range keys from in-memory index tree' (duration: 578.704129ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:35.451005Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:34.062624Z","time spent":"1.388362623s","remote":"127.0.0.1:59690","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":1,"response size":155,"request content":"key:\"/registry/masterleases/199.19.213.83\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:35.451233Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"579.013445ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:49:35.451335Z","caller":"traceutil/trace.go:171","msg":"trace[36314105] linearizableReadLoop","detail":"{readStateIndex:1043; appliedIndex:1042; }","duration":"579.260029ms","start":"2026-03-17T20:49:34.872059Z","end":"2026-03-17T20:49:35.451319Z","steps":["trace[36314105] 'read index received' (duration: 1.570217ms)","trace[36314105] 'applied index is now lower than readState.Index' (duration: 577.688802ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:35.451447Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"694.368833ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:49:35.451484Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"586.333006ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"warn","ts":"2026-03-17T20:49:35.451475Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.326496055s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces/kube-system\" limit:1 ","response":"range_response_count:1 size:351"} {"level":"info","ts":"2026-03-17T20:49:35.451522Z","caller":"traceutil/trace.go:171","msg":"trace[1452099713] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:0; response_revision:935; }","duration":"586.417797ms","start":"2026-03-17T20:49:34.865094Z","end":"2026-03-17T20:49:35.451511Z","steps":["trace[1452099713] 'agreement among raft nodes before linearized reading' (duration: 586.290815ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:35.451523Z","caller":"traceutil/trace.go:171","msg":"trace[223572251] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:934; }","duration":"695.160848ms","start":"2026-03-17T20:49:34.756345Z","end":"2026-03-17T20:49:35.451505Z","steps":["trace[223572251] 'range keys from in-memory index tree' (duration: 694.146839ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:35.451568Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:34.865079Z","time spent":"586.476628ms","remote":"127.0.0.1:59748","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":65,"response size":29,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:35.451605Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.339135499s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:35.451680Z","caller":"traceutil/trace.go:171","msg":"trace[969341616] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:935; }","duration":"1.339257932s","start":"2026-03-17T20:49:34.112411Z","end":"2026-03-17T20:49:35.451669Z","steps":["trace[969341616] 'agreement among raft nodes before linearized reading' (duration: 1.339060318s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:35.451712Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:34.112389Z","time spent":"1.339313052s","remote":"127.0.0.1:59678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-17T20:49:35.451564Z","caller":"traceutil/trace.go:171","msg":"trace[2059234259] range","detail":"{range_begin:/registry/namespaces/kube-system; range_end:; response_count:1; response_revision:935; }","duration":"1.326618337s","start":"2026-03-17T20:49:34.124928Z","end":"2026-03-17T20:49:35.451546Z","steps":["trace[2059234259] 'agreement among raft nodes before linearized reading' (duration: 1.326477535s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:35.451811Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:34.124909Z","time spent":"1.326889402s","remote":"127.0.0.1:59780","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":1,"response size":373,"request content":"key:\"/registry/namespaces/kube-system\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:35.526517Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"243.070242ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces/kube-system\" limit:1 ","response":"range_response_count:1 size:351"} {"level":"info","ts":"2026-03-17T20:49:35.526619Z","caller":"traceutil/trace.go:171","msg":"trace[1635937116] range","detail":"{range_begin:/registry/namespaces/kube-system; range_end:; response_count:1; response_revision:937; }","duration":"243.186724ms","start":"2026-03-17T20:49:35.283408Z","end":"2026-03-17T20:49:35.526594Z","steps":["trace[1635937116] 'agreement among raft nodes before linearized reading' (duration: 243.003731ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:36.654739Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:37.155390Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:37.655672Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:38.155922Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:38.656705Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:38.990798Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000829501s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:49:38.990882Z","caller":"traceutil/trace.go:171","msg":"trace[2098130180] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000965653s","start":"2026-03-17T20:49:36.989898Z","end":"2026-03-17T20:49:38.990863Z","steps":["trace[2098130180] 'agreement among raft nodes before linearized reading' (duration: 2.000866071s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:38.990931Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:36.989878Z","time spent":"2.001043865s","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:49:38 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:39.157681Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:39.657899Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:40.159055Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:40.660054Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:40.991708Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000521995s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:49:40.991807Z","caller":"traceutil/trace.go:171","msg":"trace[1614497514] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000706258s","start":"2026-03-17T20:49:38.991080Z","end":"2026-03-17T20:49:40.991786Z","steps":["trace[1614497514] 'agreement among raft nodes before linearized reading' (duration: 2.000585246s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:40.991860Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:38.991065Z","time spent":"2.00078488s","remote":"127.0.0.1:59678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:49:40 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:41.160576Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:41.579141Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"5.523483767s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-17T20:49:41.661113Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:41.661405Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.999147444s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:49:41.661469Z","caller":"traceutil/trace.go:171","msg":"trace[292316911] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"4.999337797s","start":"2026-03-17T20:49:36.662112Z","end":"2026-03-17T20:49:41.661450Z","steps":["trace[292316911] 'agreement among raft nodes before linearized reading' (duration: 4.999251476s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:41.661584Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:36.662096Z","time spent":"4.999417358s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} 2026/03/17 20:49:41 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:42.162153Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:42.189048Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 2, index: 1051, vote: 0] ignored MsgPreVote from 3cf904ec39d3e333 [logterm: 2, index: 1051] at term 2: lease is not expired (remaining ticks: 9)"} {"level":"warn","ts":"2026-03-17T20:49:42.662297Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808372,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:42.692534Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.999123999s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:49:42.692761Z","caller":"traceutil/trace.go:171","msg":"trace[1684115844] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999395524s","start":"2026-03-17T20:49:37.693345Z","end":"2026-03-17T20:49:42.692740Z","steps":["trace[1684115844] 'agreement among raft nodes before linearized reading' (duration: 4.9991422s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:42.692880Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:37.693330Z","time spent":"4.999535648s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} 2026/03/17 20:49:42 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:43.000215Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000471412s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:49:43.000330Z","caller":"traceutil/trace.go:171","msg":"trace[1949643594] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000673635s","start":"2026-03-17T20:49:40.999630Z","end":"2026-03-17T20:49:43.000304Z","steps":["trace[1949643594] 'agreement among raft nodes before linearized reading' (duration: 2.000476872s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:43.000403Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:40.999616Z","time spent":"2.000765427s","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:49:43.126363Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:36.126052Z","time spent":"7.000304712s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-17T20:49:43.155610Z","caller":"etcdserver/v3_server.go:932","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"} {"level":"warn","ts":"2026-03-17T20:49:43.155765Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.001131585s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllers/\" range_end:\"/registry/controllers0\" count_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-17T20:49:43.155800Z","caller":"traceutil/trace.go:171","msg":"trace[515652211] range","detail":"{range_begin:/registry/controllers/; range_end:/registry/controllers0; }","duration":"7.001216737s","start":"2026-03-17T20:49:36.154569Z","end":"2026-03-17T20:49:43.155786Z","steps":["trace[515652211] 'agreement among raft nodes before linearized reading' (duration: 7.001163196s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:43.155833Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:36.154551Z","time spent":"7.001273468s","remote":"127.0.0.1:59858","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":0,"response size":0,"request content":"key:\"/registry/controllers/\" range_end:\"/registry/controllers0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:43.236412Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:36.235643Z","time spent":"7.000760815s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-17T20:49:43.656703Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808376,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:43.912043Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 is starting a new election at term 2"} {"level":"info","ts":"2026-03-17T20:49:43.912115Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became pre-candidate at term 2"} {"level":"info","ts":"2026-03-17T20:49:43.912135Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 received MsgPreVoteResp from 5a110d012eba26a2 at term 2"} {"level":"info","ts":"2026-03-17T20:49:43.912160Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 2, index: 1051] sent MsgPreVote request to 3cf904ec39d3e333 at term 2"} {"level":"info","ts":"2026-03-17T20:49:43.912171Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 2, index: 1051] sent MsgPreVote request to 77297b5b7846c4bf at term 2"} {"level":"info","ts":"2026-03-17T20:49:43.912183Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: 5a110d012eba26a2 lost leader 3cf904ec39d3e333 at term 2"} {"level":"info","ts":"2026-03-17T20:49:43.913311Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 received MsgPreVoteResp from 3cf904ec39d3e333 at term 2"} {"level":"info","ts":"2026-03-17T20:49:43.913334Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 has received 2 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-17T20:49:43.913349Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became candidate at term 3"} {"level":"info","ts":"2026-03-17T20:49:43.913358Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 received MsgVoteResp from 5a110d012eba26a2 at term 3"} {"level":"info","ts":"2026-03-17T20:49:43.913369Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 2, index: 1051] sent MsgVote request to 3cf904ec39d3e333 at term 3"} {"level":"info","ts":"2026-03-17T20:49:43.913378Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 2, index: 1051] sent MsgVote request to 77297b5b7846c4bf at term 3"} {"level":"warn","ts":"2026-03-17T20:49:44.157119Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808376,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:44.362105Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 received MsgVoteResp from 3cf904ec39d3e333 at term 3"} {"level":"info","ts":"2026-03-17T20:49:44.362180Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 has received 2 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-17T20:49:44.362261Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became leader at term 3"} {"level":"info","ts":"2026-03-17T20:49:44.362283Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: 5a110d012eba26a2 elected leader 5a110d012eba26a2 at term 3"} {"level":"warn","ts":"2026-03-17T20:49:44.362517Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.663616106s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/\" range_end:\"/registry/apiextensions.k8s.io/customresourcedefinitions0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:44.362597Z","caller":"traceutil/trace.go:171","msg":"trace[1540022611] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions/; range_end:/registry/apiextensions.k8s.io/customresourcedefinitions0; }","duration":"6.663744078s","start":"2026-03-17T20:49:37.698836Z","end":"2026-03-17T20:49:44.362580Z","steps":["trace[1540022611] 'agreement among raft nodes before linearized reading' (duration: 6.663638296s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.362633Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.605481937s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-17T20:49:44.362672Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"8.184357485s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:44.362695Z","caller":"traceutil/trace.go:171","msg":"trace[118995073] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; }","duration":"8.184424816s","start":"2026-03-17T20:49:36.178264Z","end":"2026-03-17T20:49:44.362689Z","steps":["trace[118995073] 'agreement among raft nodes before linearized reading' (duration: 8.184400426s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.362642Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:37.698825Z","time spent":"6.663805329s","remote":"127.0.0.1:59714","response type":"/etcdserverpb.KV/Range","request count":0,"request size":120,"response count":0,"response size":0,"request content":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/\" range_end:\"/registry/apiextensions.k8s.io/customresourcedefinitions0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:44.362720Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:36.178243Z","time spent":"8.184470957s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-17T20:49:44.362693Z","caller":"traceutil/trace.go:171","msg":"trace[1002075547] range","detail":"{range_begin:; range_end:; }","duration":"7.605545578s","start":"2026-03-17T20:49:36.757135Z","end":"2026-03-17T20:49:44.362681Z","steps":["trace[1002075547] 'agreement among raft nodes before linearized reading' (duration: 7.605480597s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.362725Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.443495923s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"error","ts":"2026-03-17T20:49:44.362781Z","caller":"etcdhttp/health.go:370","msg":"Health check error","path":"/readyz","reason":"[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: leader changed\n[+]non_learner ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHttpEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:370\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-17T20:49:44.363072Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.36842839s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:44.363059Z","caller":"traceutil/trace.go:171","msg":"trace[12669663] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; }","duration":"7.443579985s","start":"2026-03-17T20:49:36.919193Z","end":"2026-03-17T20:49:44.362773Z","steps":["trace[12669663] 'agreement among raft nodes before linearized reading' (duration: 7.443494153s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.363195Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.581982446s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/\" range_end:\"/registry/configmaps0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-17T20:49:44.363242Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:36.919040Z","time spent":"7.444184487s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-17T20:49:44.363254Z","caller":"traceutil/trace.go:171","msg":"trace[1863533334] range","detail":"{range_begin:/registry/configmaps/; range_end:/registry/configmaps0; }","duration":"1.582092668s","start":"2026-03-17T20:49:42.781151Z","end":"2026-03-17T20:49:44.363244Z","steps":["trace[1863533334] 'agreement among raft nodes before linearized reading' (duration: 1.582025077s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.363290Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:42.781128Z","time spent":"1.582150119s","remote":"127.0.0.1:59770","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/configmaps/\" range_end:\"/registry/configmaps0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:44.363317Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.268074173s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/jobs/\" range_end:\"/registry/jobs0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-17T20:49:44.363325Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.362311678s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:44.363166Z","caller":"traceutil/trace.go:171","msg":"trace[220676859] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4; range_end:; }","duration":"3.368573263s","start":"2026-03-17T20:49:40.994576Z","end":"2026-03-17T20:49:44.363150Z","steps":["trace[220676859] 'agreement among raft nodes before linearized reading' (duration: 3.368438051s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:44.363377Z","caller":"traceutil/trace.go:171","msg":"trace[2071561732] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.362392739s","start":"2026-03-17T20:49:43.000971Z","end":"2026-03-17T20:49:44.363363Z","steps":["trace[2071561732] 'agreement among raft nodes before linearized reading' (duration: 1.362338778s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.363410Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:40.994560Z","time spent":"3.368839338s","remote":"127.0.0.1:59748","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":0,"response size":0,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:44.363420Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:43.000950Z","time spent":"1.36245753s","remote":"127.0.0.1:59678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-17T20:49:44.363345Z","caller":"traceutil/trace.go:171","msg":"trace[599331913] range","detail":"{range_begin:/registry/jobs/; range_end:/registry/jobs0; }","duration":"3.268131085s","start":"2026-03-17T20:49:41.095206Z","end":"2026-03-17T20:49:44.363337Z","steps":["trace[599331913] 'agreement among raft nodes before linearized reading' (duration: 3.268100384s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.363476Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:41.095190Z","time spent":"3.268280027s","remote":"127.0.0.1:59878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":0,"request content":"key:\"/registry/jobs/\" range_end:\"/registry/jobs0\" count_only:true "} {"level":"info","ts":"2026-03-17T20:49:44.601824Z","caller":"etcdserver/v3_server.go:912","msg":"first commit in current term: resending ReadIndex request"} {"level":"info","ts":"2026-03-17T20:49:44.617136Z","caller":"traceutil/trace.go:171","msg":"trace[2044679781] linearizableReadLoop","detail":"{readStateIndex:1052; appliedIndex:1052; }","duration":"254.680554ms","start":"2026-03-17T20:49:44.362441Z","end":"2026-03-17T20:49:44.617122Z","steps":["trace[2044679781] 'read index received' (duration: 254.676844ms)","trace[2044679781] 'applied index is now lower than readState.Index' (duration: 2.85µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:44.617253Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"307.670961ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumes/\" range_end:\"/registry/persistentvolumes0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:44.617276Z","caller":"traceutil/trace.go:171","msg":"trace[225408716] range","detail":"{range_begin:/registry/persistentvolumes/; range_end:/registry/persistentvolumes0; response_count:0; response_revision:942; }","duration":"307.742582ms","start":"2026-03-17T20:49:44.309528Z","end":"2026-03-17T20:49:44.617270Z","steps":["trace[225408716] 'agreement among raft nodes before linearized reading' (duration: 307.678071ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.617307Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:44.309510Z","time spent":"307.789613ms","remote":"127.0.0.1:59810","response type":"/etcdserverpb.KV/Range","request count":0,"request size":62,"response count":0,"response size":27,"request content":"key:\"/registry/persistentvolumes/\" range_end:\"/registry/persistentvolumes0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:44.617384Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"554.282843ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.19.213.83\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-17T20:49:44.617498Z","caller":"traceutil/trace.go:171","msg":"trace[1931720661] range","detail":"{range_begin:/registry/masterleases/199.19.213.83; range_end:; response_count:1; response_revision:942; }","duration":"554.412385ms","start":"2026-03-17T20:49:44.063064Z","end":"2026-03-17T20:49:44.617477Z","steps":["trace[1931720661] 'agreement among raft nodes before linearized reading' (duration: 554.18374ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:44.617552Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:44.063001Z","time spent":"554.537757ms","remote":"127.0.0.1:59690","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":1,"response size":155,"request content":"key:\"/registry/masterleases/199.19.213.83\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:44.751156Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:44.368476Z","time spent":"382.676887ms","remote":"127.0.0.1:59748","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-17T20:49:45.117877Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:45.518252Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"77297b5b7846c4bf","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"591.732161ms"} {"level":"warn","ts":"2026-03-17T20:49:45.518363Z","caller":"etcdserver/raft.go:427","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"591.845863ms"} {"level":"warn","ts":"2026-03-17T20:49:45.618067Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:46.118264Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:46.372878Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.00040239s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:49:46.373082Z","caller":"traceutil/trace.go:171","msg":"trace[844561444] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000657635s","start":"2026-03-17T20:49:44.372404Z","end":"2026-03-17T20:49:46.373062Z","steps":["trace[844561444] 'agreement among raft nodes before linearized reading' (duration: 2.000430291s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:46.373152Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:44.372391Z","time spent":"2.000747706s","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:49:46.619117Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:47.011740Z","logger":"raft","caller":"etcdserver/zap_raft.go:85","msg":"5a110d012eba26a2 stepped down to follower since quorum is not active"} {"level":"info","ts":"2026-03-17T20:49:47.011790Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became follower at term 3"} {"level":"info","ts":"2026-03-17T20:49:47.011802Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: 5a110d012eba26a2 lost leader 5a110d012eba26a2 at term 3"} {"level":"warn","ts":"2026-03-17T20:49:47.119587Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:47.119690Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 no leader at term 3; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:49:47.596929Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000198893s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:49:47.597000Z","caller":"traceutil/trace.go:171","msg":"trace[1486389839] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000285294s","start":"2026-03-17T20:49:45.596698Z","end":"2026-03-17T20:49:47.596983Z","steps":["trace[1486389839] 'agreement among raft nodes before linearized reading' (duration: 2.000196152s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:47.597037Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:45.596619Z","time spent":"2.000410637s","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:49:47 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:47.620407Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:47.620577Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 no leader at term 3; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:49:48.121374Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:48.121639Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 no leader at term 3; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:49:48.374013Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.00002049s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:49:48.374123Z","caller":"traceutil/trace.go:171","msg":"trace[1721850692] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000194923s","start":"2026-03-17T20:49:46.373905Z","end":"2026-03-17T20:49:48.374100Z","steps":["trace[1721850692] 'agreement among raft nodes before linearized reading' (duration: 2.000048781s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:48.374193Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:46.373882Z","time spent":"2.000290775s","remote":"127.0.0.1:59678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:49:48 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-17T20:49:48.612037Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 is starting a new election at term 3"} {"level":"info","ts":"2026-03-17T20:49:48.612105Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became pre-candidate at term 3"} {"level":"info","ts":"2026-03-17T20:49:48.612125Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 received MsgPreVoteResp from 5a110d012eba26a2 at term 3"} {"level":"info","ts":"2026-03-17T20:49:48.612151Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 3, index: 1068] sent MsgPreVote request to 3cf904ec39d3e333 at term 3"} {"level":"info","ts":"2026-03-17T20:49:48.612166Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 3, index: 1068] sent MsgPreVote request to 77297b5b7846c4bf at term 3"} {"level":"warn","ts":"2026-03-17T20:49:48.622286Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:49.123408Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:49.623915Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:50.124708Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:50.136109Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:43.131820Z","time spent":"7.004281712s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"info","ts":"2026-03-17T20:49:50.211336Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 is starting a new election at term 3"} {"level":"info","ts":"2026-03-17T20:49:50.211432Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became pre-candidate at term 3"} {"level":"info","ts":"2026-03-17T20:49:50.211448Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 received MsgPreVoteResp from 5a110d012eba26a2 at term 3"} {"level":"info","ts":"2026-03-17T20:49:50.211476Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 3, index: 1068] sent MsgPreVote request to 3cf904ec39d3e333 at term 3"} {"level":"info","ts":"2026-03-17T20:49:50.211486Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 3, index: 1068] sent MsgPreVote request to 77297b5b7846c4bf at term 3"} {"level":"warn","ts":"2026-03-17T20:49:50.242297Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:43.242029Z","time spent":"7.000255986s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-17T20:49:50.383426Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.999835052s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:49:50.383642Z","caller":"traceutil/trace.go:171","msg":"trace[30345457] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000114457s","start":"2026-03-17T20:49:48.383512Z","end":"2026-03-17T20:49:50.383626Z","steps":["trace[30345457] 'agreement among raft nodes before linearized reading' (duration: 1.999880653s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:50.383736Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:48.383485Z","time spent":"2.000241979s","remote":"127.0.0.1:59686","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:49:50.628218Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:50.921659Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.000533496s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:49:50.921731Z","caller":"traceutil/trace.go:171","msg":"trace[1518797698] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"5.000663848s","start":"2026-03-17T20:49:45.921055Z","end":"2026-03-17T20:49:50.921719Z","steps":["trace[1518797698] 'agreement among raft nodes before linearized reading' (duration: 5.000563927s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:50.921771Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:45.921041Z","time spent":"5.000719249s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} 2026/03/17 20:49:50 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:51.128338Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808381,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:49:51.136816Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.999827567s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:49:51.140917Z","caller":"traceutil/trace.go:171","msg":"trace[17118695] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999936529s","start":"2026-03-17T20:49:46.136922Z","end":"2026-03-17T20:49:51.136859Z","steps":["trace[17118695] 'agreement among raft nodes before linearized reading' (duration: 4.999859778s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:51.140991Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:46.136905Z","time spent":"5.004070151s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} 2026/03/17 20:49:51 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:49:51.617229Z","caller":"etcdserver/v3_server.go:932","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"} {"level":"warn","ts":"2026-03-17T20:49:51.617373Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.071792645s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-k8r54\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-17T20:49:51.617653Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.253318447s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-17T20:49:51.617836Z","caller":"traceutil/trace.go:171","msg":"trace[25071970] range","detail":"{range_begin:; range_end:; }","duration":"7.253690373s","start":"2026-03-17T20:49:44.364128Z","end":"2026-03-17T20:49:51.617819Z","steps":["trace[25071970] 'agreement among raft nodes before linearized reading' (duration: 7.253298227s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:51.617910Z","caller":"traceutil/trace.go:171","msg":"trace[736267321] range","detail":"{range_begin:/registry/pods/kube-system/cilium-k8r54; range_end:; }","duration":"7.07205429s","start":"2026-03-17T20:49:44.545413Z","end":"2026-03-17T20:49:51.617467Z","steps":["trace[736267321] 'agreement among raft nodes before linearized reading' (duration: 7.071884657s)"],"step_count":1} {"level":"error","ts":"2026-03-17T20:49:51.617888Z","caller":"etcdhttp/health.go:370","msg":"Health check error","path":"/readyz","reason":"[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: request timed out\n[+]non_learner ok\n[+]data_corruption ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHttpEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:370\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-17T20:49:51.618034Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:44.545350Z","time spent":"7.072599751s","remote":"127.0.0.1:59840","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":0,"response size":0,"request content":"key:\"/registry/pods/kube-system/cilium-k8r54\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:51.618930Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:44.618807Z","time spent":"7.000120148s","remote":"127.0.0.1:59690","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-17T20:49:51.752912Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:44.752467Z","time spent":"7.000440499s","remote":"127.0.0.1:59748","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"info","ts":"2026-03-17T20:49:51.811819Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 is starting a new election at term 3"} {"level":"info","ts":"2026-03-17T20:49:51.811947Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became pre-candidate at term 3"} {"level":"info","ts":"2026-03-17T20:49:51.811978Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 received MsgPreVoteResp from 5a110d012eba26a2 at term 3"} {"level":"info","ts":"2026-03-17T20:49:51.811998Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 3, index: 1068] sent MsgPreVote request to 3cf904ec39d3e333 at term 3"} {"level":"info","ts":"2026-03-17T20:49:51.812007Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 3, index: 1068] sent MsgPreVote request to 77297b5b7846c4bf at term 3"} {"level":"warn","ts":"2026-03-17T20:49:52.118638Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808387,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:52.362779Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 3, index: 1068, vote: 5a110d012eba26a2] cast MsgPreVote for 77297b5b7846c4bf [logterm: 3, index: 1068] at term 3"} {"level":"info","ts":"2026-03-17T20:49:52.363480Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [term: 3] received a MsgVote message with higher term from 77297b5b7846c4bf [term: 4]"} {"level":"info","ts":"2026-03-17T20:49:52.363551Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 became follower at term 4"} {"level":"info","ts":"2026-03-17T20:49:52.363577Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"5a110d012eba26a2 [logterm: 3, index: 1068, vote: 0] cast MsgVote for 77297b5b7846c4bf [logterm: 3, index: 1068] at term 4"} {"level":"info","ts":"2026-03-17T20:49:52.367691Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: 5a110d012eba26a2 elected leader 77297b5b7846c4bf at term 4"} {"level":"warn","ts":"2026-03-17T20:49:52.369882Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.550818596s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:52.369939Z","caller":"traceutil/trace.go:171","msg":"trace[816875260] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; }","duration":"5.550921689s","start":"2026-03-17T20:49:46.819006Z","end":"2026-03-17T20:49:52.369928Z","steps":["trace[816875260] 'agreement among raft nodes before linearized reading' (duration: 5.550846607s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.369968Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:46.818989Z","time spent":"5.550972179s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:52.370004Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.129670151s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/\" range_end:\"/registry/serviceaccounts0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:52.370018Z","caller":"traceutil/trace.go:171","msg":"trace[208813898] range","detail":"{range_begin:/registry/serviceaccounts/; range_end:/registry/serviceaccounts0; }","duration":"1.129711101s","start":"2026-03-17T20:49:51.240303Z","end":"2026-03-17T20:49:52.370014Z","steps":["trace[208813898] 'agreement among raft nodes before linearized reading' (duration: 1.129692801s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.370032Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:51.240290Z","time spent":"1.129736812s","remote":"127.0.0.1:59848","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":0,"response size":0,"request content":"key:\"/registry/serviceaccounts/\" range_end:\"/registry/serviceaccounts0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:52.370054Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.983282166s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-17T20:49:52.370276Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.142332971s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/servicecidrs/\" range_end:\"/registry/servicecidrs0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:52.370344Z","caller":"traceutil/trace.go:171","msg":"trace[1270514449] range","detail":"{range_begin:/registry/servicecidrs/; range_end:/registry/servicecidrs0; }","duration":"7.142427523s","start":"2026-03-17T20:49:45.227906Z","end":"2026-03-17T20:49:52.370333Z","steps":["trace[1270514449] 'agreement among raft nodes before linearized reading' (duration: 7.142330071s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.370376Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:45.227789Z","time spent":"7.142579755s","remote":"127.0.0.1:59960","response type":"/etcdserverpb.KV/Range","request count":0,"request size":52,"response count":0,"response size":0,"request content":"key:\"/registry/servicecidrs/\" range_end:\"/registry/servicecidrs0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:52.370414Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.727998585s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/\" range_end:\"/registry/endpointslices0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:52.370430Z","caller":"traceutil/trace.go:171","msg":"trace[1219525535] range","detail":"{range_begin:/registry/endpointslices/; range_end:/registry/endpointslices0; }","duration":"6.728016225s","start":"2026-03-17T20:49:45.642410Z","end":"2026-03-17T20:49:52.370426Z","steps":["trace[1219525535] 'agreement among raft nodes before linearized reading' (duration: 6.727998155s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.370444Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:45.642372Z","time spent":"6.728068116s","remote":"127.0.0.1:59926","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/endpointslices/\" range_end:\"/registry/endpointslices0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:52.370472Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.798832988s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/poddisruptionbudgets/\" range_end:\"/registry/poddisruptionbudgets0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:52.370484Z","caller":"traceutil/trace.go:171","msg":"trace[1657091993] range","detail":"{range_begin:/registry/poddisruptionbudgets/; range_end:/registry/poddisruptionbudgets0; }","duration":"6.798873798s","start":"2026-03-17T20:49:45.571607Z","end":"2026-03-17T20:49:52.370480Z","steps":["trace[1657091993] 'agreement among raft nodes before linearized reading' (duration: 6.798860228s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.370506Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:45.571594Z","time spent":"6.798904478s","remote":"127.0.0.1:59980","response type":"/etcdserverpb.KV/Range","request count":0,"request size":68,"response count":0,"response size":0,"request content":"key:\"/registry/poddisruptionbudgets/\" range_end:\"/registry/poddisruptionbudgets0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:52.370916Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.035053241s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:52.370944Z","caller":"traceutil/trace.go:171","msg":"trace[387016107] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; }","duration":"5.035112462s","start":"2026-03-17T20:49:47.335826Z","end":"2026-03-17T20:49:52.370938Z","steps":["trace[387016107] 'agreement among raft nodes before linearized reading' (duration: 5.035082571s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.370960Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:47.335808Z","time spent":"5.035147592s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:52.370986Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.605064911s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:52.371000Z","caller":"traceutil/trace.go:171","msg":"trace[2111617790] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; }","duration":"3.605108671s","start":"2026-03-17T20:49:48.765888Z","end":"2026-03-17T20:49:52.370997Z","steps":["trace[2111617790] 'agreement among raft nodes before linearized reading' (duration: 3.605092681s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.371013Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:48.765867Z","time spent":"3.605141392s","remote":"127.0.0.1:60198","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":0,"response size":0,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true "} {"level":"warn","ts":"2026-03-17T20:49:52.371039Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.498833928s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/networkpolicies/\" range_end:\"/registry/networkpolicies0\" count_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:49:52.371053Z","caller":"traceutil/trace.go:171","msg":"trace[1611547545] range","detail":"{range_begin:/registry/networkpolicies/; range_end:/registry/networkpolicies0; }","duration":"4.498881408s","start":"2026-03-17T20:49:47.872169Z","end":"2026-03-17T20:49:52.371050Z","steps":["trace[1611547545] 'agreement among raft nodes before linearized reading' (duration: 4.498865328s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.371066Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:47.872151Z","time spent":"4.498910598s","remote":"127.0.0.1:59928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":0,"response size":0,"request content":"key:\"/registry/networkpolicies/\" range_end:\"/registry/networkpolicies0\" count_only:true "} {"level":"info","ts":"2026-03-17T20:49:52.370802Z","caller":"traceutil/trace.go:171","msg":"trace[1709037270] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.983313747s","start":"2026-03-17T20:49:50.386747Z","end":"2026-03-17T20:49:52.370061Z","steps":["trace[1709037270] 'agreement among raft nodes before linearized reading' (duration: 1.983303167s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.371522Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:50.386733Z","time spent":"1.984779493s","remote":"127.0.0.1:59678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-17T20:49:52.377761Z","caller":"etcdserver/v3_server.go:912","msg":"first commit in current term: resending ReadIndex request"} {"level":"warn","ts":"2026-03-17T20:49:52.385014Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"625.944412ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-17T20:49:52.385092Z","caller":"traceutil/trace.go:171","msg":"trace[2019570732] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4; range_end:; response_count:1; response_revision:949; }","duration":"626.051664ms","start":"2026-03-17T20:49:51.759020Z","end":"2026-03-17T20:49:52.385071Z","steps":["trace[2019570732] 'agreement among raft nodes before linearized reading' (duration: 625.879731ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.385136Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:51.759010Z","time spent":"626.116175ms","remote":"127.0.0.1:59748","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":816,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-3.189dbbff347393d4\" limit:1 "} {"level":"info","ts":"2026-03-17T20:49:52.385295Z","caller":"traceutil/trace.go:171","msg":"trace[1442292832] transaction","detail":"{read_only:false; number_of_response:1; response_revision:949; }","duration":"2.126994363s","start":"2026-03-17T20:49:50.258277Z","end":"2026-03-17T20:49:52.385271Z","steps":["trace[1442292832] 'process raft request' (duration: 2.126952842s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.385459Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:50.258257Z","time spent":"2.127115975s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":49,"response count":0,"response size":584,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:49:52.385566Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"757.597697ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-k8r54\" limit:1 ","response":"range_response_count:1 size:18330"} {"level":"info","ts":"2026-03-17T20:49:52.385623Z","caller":"traceutil/trace.go:171","msg":"trace[668977031] range","detail":"{range_begin:/registry/pods/kube-system/cilium-k8r54; range_end:; response_count:1; response_revision:949; }","duration":"757.683638ms","start":"2026-03-17T20:49:51.627923Z","end":"2026-03-17T20:49:52.385606Z","steps":["trace[668977031] 'agreement among raft nodes before linearized reading' (duration: 757.412692ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.385681Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:51.627911Z","time spent":"757.759509ms","remote":"127.0.0.1:59840","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18352,"request content":"key:\"/registry/pods/kube-system/cilium-k8r54\" limit:1 "} {"level":"info","ts":"2026-03-17T20:49:52.385728Z","caller":"traceutil/trace.go:171","msg":"trace[1976672184] transaction","detail":"{read_only:false; number_of_response:1; response_revision:949; }","duration":"2.24074798s","start":"2026-03-17T20:49:50.144969Z","end":"2026-03-17T20:49:52.385717Z","steps":["trace[1976672184] 'process raft request' (duration: 2.240007137s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.385781Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:50.144914Z","time spent":"2.240841272s","remote":"127.0.0.1:59914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":69,"response count":0,"response size":724,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:49:52.386171Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"766.156867ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:52.387528Z","caller":"traceutil/trace.go:171","msg":"trace[659284913] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:949; }","duration":"766.270429ms","start":"2026-03-17T20:49:51.619953Z","end":"2026-03-17T20:49:52.386223Z","steps":["trace[659284913] 'agreement among raft nodes before linearized reading' (duration: 766.119807ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:52.834877Z","caller":"traceutil/trace.go:171","msg":"trace[1408380405] transaction","detail":"{read_only:false; response_revision:971; number_of_response:1; }","duration":"389.063793ms","start":"2026-03-17T20:49:52.445795Z","end":"2026-03-17T20:49:52.834859Z","steps":["trace[1408380405] 'process raft request' (duration: 388.915041ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.835012Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:52.445770Z","time spent":"389.172155ms","remote":"127.0.0.1:59840","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7786,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:49:52.835154Z","caller":"traceutil/trace.go:171","msg":"trace[572034035] transaction","detail":"{read_only:false; response_revision:972; number_of_response:1; }","duration":"372.293267ms","start":"2026-03-17T20:49:52.462832Z","end":"2026-03-17T20:49:52.835125Z","steps":["trace[572034035] 'process raft request' (duration: 371.991892ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:52.835271Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:52.462800Z","time spent":"372.41739ms","remote":"127.0.0.1:59748","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":778,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-17T20:49:54.033181Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"276.7265ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:54.033275Z","caller":"traceutil/trace.go:171","msg":"trace[1960301915] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1031; }","duration":"276.836622ms","start":"2026-03-17T20:49:53.756417Z","end":"2026-03-17T20:49:54.033254Z","steps":["trace[1960301915] 'range keys from in-memory index tree' (duration: 276.681549ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:54.095048Z","caller":"traceutil/trace.go:171","msg":"trace[1575071181] linearizableReadLoop","detail":"{readStateIndex:1160; appliedIndex:1160; }","duration":"358.597442ms","start":"2026-03-17T20:49:53.736415Z","end":"2026-03-17T20:49:54.095012Z","steps":["trace[1575071181] 'read index received' (duration: 358.581562ms)","trace[1575071181] 'applied index is now lower than readState.Index' (duration: 14.09µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:54.095462Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"359.001979ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-k8r54\" limit:1 ","response":"range_response_count:1 size:19215"} {"level":"info","ts":"2026-03-17T20:49:54.095433Z","caller":"traceutil/trace.go:171","msg":"trace[1612921832] transaction","detail":"{read_only:false; response_revision:1033; number_of_response:1; }","duration":"357.959012ms","start":"2026-03-17T20:49:53.737439Z","end":"2026-03-17T20:49:54.095398Z","steps":["trace[1612921832] 'process raft request' (duration: 357.591666ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:54.095614Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:53.737429Z","time spent":"358.111295ms","remote":"127.0.0.1:59748","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":759,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:49:54.095497Z","caller":"traceutil/trace.go:171","msg":"trace[473471847] range","detail":"{range_begin:/registry/pods/kube-system/cilium-k8r54; range_end:; response_count:1; response_revision:1034; }","duration":"359.098712ms","start":"2026-03-17T20:49:53.736388Z","end":"2026-03-17T20:49:54.095486Z","steps":["trace[473471847] 'agreement among raft nodes before linearized reading' (duration: 358.915908ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:54.096030Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:53.736371Z","time spent":"359.58507ms","remote":"127.0.0.1:59840","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":19237,"request content":"key:\"/registry/pods/kube-system/cilium-k8r54\" limit:1 "} {"level":"info","ts":"2026-03-17T20:49:54.595613Z","caller":"traceutil/trace.go:171","msg":"trace[1971971201] linearizableReadLoop","detail":"{readStateIndex:1177; appliedIndex:1177; }","duration":"290.137433ms","start":"2026-03-17T20:49:54.305453Z","end":"2026-03-17T20:49:54.595590Z","steps":["trace[1971971201] 'read index received' (duration: 290.130523ms)","trace[1971971201] 'applied index is now lower than readState.Index' (duration: 6.05µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:49:54.595790Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"290.306968ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:54.595830Z","caller":"traceutil/trace.go:171","msg":"trace[1237846012] range","detail":"{range_begin:/registry/horizontalpodautoscalers/; range_end:/registry/horizontalpodautoscalers0; response_count:0; response_revision:1048; }","duration":"290.415519ms","start":"2026-03-17T20:49:54.305401Z","end":"2026-03-17T20:49:54.595816Z","steps":["trace[1237846012] 'agreement among raft nodes before linearized reading' (duration: 290.320047ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:54.597564Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"188.505262ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:54.597640Z","caller":"traceutil/trace.go:171","msg":"trace[1598600014] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1048; }","duration":"188.614904ms","start":"2026-03-17T20:49:54.408992Z","end":"2026-03-17T20:49:54.597607Z","steps":["trace[1598600014] 'agreement among raft nodes before linearized reading' (duration: 188.501682ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:57.035846Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2783960132499808465,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:49:57.191650Z","caller":"traceutil/trace.go:171","msg":"trace[65805785] transaction","detail":"{read_only:false; response_revision:1082; number_of_response:1; }","duration":"659.024819ms","start":"2026-03-17T20:49:56.532603Z","end":"2026-03-17T20:49:57.191627Z","steps":["trace[65805785] 'process raft request' (duration: 658.845646ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:57.191796Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:56.532588Z","time spent":"659.138121ms","remote":"127.0.0.1:59714","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3104,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:49:57.201124Z","caller":"traceutil/trace.go:171","msg":"trace[1413002939] linearizableReadLoop","detail":"{readStateIndex:1210; appliedIndex:1211; }","duration":"665.999953ms","start":"2026-03-17T20:49:56.535107Z","end":"2026-03-17T20:49:57.201107Z","steps":["trace[1413002939] 'read index received' (duration: 665.994973ms)","trace[1413002939] 'applied index is now lower than readState.Index' (duration: 3.98µs)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:49:57.201338Z","caller":"traceutil/trace.go:171","msg":"trace[253563330] transaction","detail":"{read_only:false; response_revision:1083; number_of_response:1; }","duration":"666.606053ms","start":"2026-03-17T20:49:56.534707Z","end":"2026-03-17T20:49:57.201313Z","steps":["trace[253563330] 'process raft request' (duration: 666.359369ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:57.201623Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"670.114405ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/ciliuml2announcementpolicies.cilium.io\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:49:57.201662Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:56.534696Z","time spent":"666.915258ms","remote":"127.0.0.1:59714","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4234,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:49:57.201760Z","caller":"traceutil/trace.go:171","msg":"trace[1143526872] transaction","detail":"{read_only:false; response_revision:1085; number_of_response:1; }","duration":"474.769854ms","start":"2026-03-17T20:49:56.726978Z","end":"2026-03-17T20:49:57.201748Z","steps":["trace[1143526872] 'process raft request' (duration: 474.229744ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:57.201780Z","caller":"traceutil/trace.go:171","msg":"trace[1882983820] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions/ciliuml2announcementpolicies.cilium.io; range_end:; response_count:0; response_revision:1084; }","duration":"670.207267ms","start":"2026-03-17T20:49:56.531461Z","end":"2026-03-17T20:49:57.201668Z","steps":["trace[1882983820] 'agreement among raft nodes before linearized reading' (duration: 669.881752ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:49:57.201508Z","caller":"traceutil/trace.go:171","msg":"trace[587747158] transaction","detail":"{read_only:false; response_revision:1084; number_of_response:1; }","duration":"593.81771ms","start":"2026-03-17T20:49:56.607651Z","end":"2026-03-17T20:49:57.201468Z","steps":["trace[587747158] 'process raft request' (duration: 593.515525ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:57.201838Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:56.531447Z","time spent":"670.37639ms","remote":"127.0.0.1:59714","response type":"/etcdserverpb.KV/Range","request count":0,"request size":99,"response count":0,"response size":27,"request content":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/ciliuml2announcementpolicies.cilium.io\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:49:57.201851Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:56.607617Z","time spent":"594.217127ms","remote":"127.0.0.1:59748","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":788,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-17T20:49:57.203969Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:56.726957Z","time spent":"474.860895ms","remote":"127.0.0.1:59714","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3202,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-17T20:49:57.225652Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"468.046724ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:57.225722Z","caller":"traceutil/trace.go:171","msg":"trace[689516747] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1089; }","duration":"468.125385ms","start":"2026-03-17T20:49:56.757581Z","end":"2026-03-17T20:49:57.225706Z","steps":["trace[689516747] 'agreement among raft nodes before linearized reading' (duration: 468.003133ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:57.225664Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"300.017184ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/ciliumnodes.cilium.io\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:49:57.225842Z","caller":"traceutil/trace.go:171","msg":"trace[1290465717] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions/ciliumnodes.cilium.io; range_end:; response_count:0; response_revision:1089; }","duration":"300.248998ms","start":"2026-03-17T20:49:56.925574Z","end":"2026-03-17T20:49:57.225823Z","steps":["trace[1290465717] 'agreement among raft nodes before linearized reading' (duration: 300.006504ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:49:57.225902Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-17T20:49:56.925555Z","time spent":"300.3347ms","remote":"127.0.0.1:59714","response type":"/etcdserverpb.KV/Range","request count":0,"request size":82,"response count":0,"response size":27,"request content":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/ciliumnodes.cilium.io\" limit:1 "}