{"level":"warn","ts":"2026-03-30T16:47:55.225889Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"warn","ts":"2026-03-30T16:47:55.226050Z","caller":"etcdmain/config.go:270","msg":"--snapshot-count is deprecated in 3.6 and will be decommissioned in 3.7."} {"level":"info","ts":"2026-03-30T16:47:55.226061Z","caller":"etcdmain/etcd.go:64","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.204.45.227:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--feature-gates=InitialCorruptCheck=true","--initial-advertise-peer-urls=https://199.204.45.227:2380","--initial-cluster=controller-1=https://199.19.213.3:2380,controller-2=https://199.204.45.227:2380","--initial-cluster-state=existing","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.204.45.227:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.204.45.227:2380","--name=controller-2","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--watch-progress-notify-interval=5s"]} {"level":"info","ts":"2026-03-30T16:47:55.226146Z","caller":"etcdmain/etcd.go:124","msg":"Initialize and start etcd server","data-dir":"/var/lib/etcd","dir-type":"empty"} {"level":"warn","ts":"2026-03-30T16:47:55.226164Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-30T16:47:55.226174Z","caller":"embed/etcd.go:138","msg":"configuring peer listeners","listen-peer-urls":["https://199.204.45.227:2380"]} {"level":"info","ts":"2026-03-30T16:47:55.226203Z","caller":"embed/etcd.go:544","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-30T16:47:55.226629Z","caller":"embed/etcd.go:146","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.227:2379"]} {"level":"info","ts":"2026-03-30T16:47:55.226728Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.6.4","git-sha":"5400cdc","go-version":"go1.23.11","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-2","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.204.45.227:2380"],"listen-peer-urls":["https://199.204.45.227:2380"],"advertise-client-urls":["https://199.204.45.227:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.227:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"experimental-local-address":"","cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://199.19.213.3:2380,controller-2=https://199.204.45.227:2380","initial-cluster-state":"existing","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"feature-gates":"InitialCorruptCheck=true","initial-corrupt-check":false,"corrupt-check-time-interval":"0s","compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","discovery-token":"","discovery-endpoints":"","discovery-dial-timeout":"2s","discovery-request-timeout":"5s","discovery-keepalive-time":"2s","discovery-keepalive-timeout":"6s","discovery-insecure-transport":true,"discovery-insecure-skip-tls-verify":false,"discovery-cert":"","discovery-key":"","discovery-cacert":"","discovery-user":"","downgrade-check-interval":"5s","max-learners":1,"v2-deprecation":"write-only"} {"level":"info","ts":"2026-03-30T16:47:55.227418Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc0004161a8}"} {"level":"info","ts":"2026-03-30T16:47:55.264896Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-30T16:47:55.265030Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"37.706835ms"} {"level":"info","ts":"2026-03-30T16:47:55.265119Z","caller":"etcdserver/bootstrap.go:220","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-03-30T16:47:55.265147Z","caller":"etcdserver/bootstrap.go:94","msg":"bootstrapping cluster"} {"level":"info","ts":"2026-03-30T16:47:55.280506Z","caller":"netutil/netutil.go:113","msg":"resolved URL Host","url":"https://199.204.45.227:2380","host":"199.204.45.227:2380","resolved-addr":"199.204.45.227:2380"} {"level":"info","ts":"2026-03-30T16:47:55.280573Z","caller":"netutil/netutil.go:113","msg":"resolved URL Host","url":"https://199.19.213.3:2380","host":"199.19.213.3:2380","resolved-addr":"199.19.213.3:2380"} {"level":"info","ts":"2026-03-30T16:47:55.282231Z","caller":"etcdserver/bootstrap.go:101","msg":"bootstrapping storage"} {"level":"info","ts":"2026-03-30T16:47:55.312180Z","caller":"etcdserver/bootstrap.go:109","msg":"bootstrapping raft"} {"level":"info","ts":"2026-03-30T16:47:55.312258Z","caller":"etcdserver/bootstrap.go:499","msg":"starting local member","local-member-id":"a8d4e8c653fa8dce","cluster-id":"7b1ae51097bedd0e"} {"level":"info","ts":"2026-03-30T16:47:55.312300Z","caller":"etcdserver/server.go:312","msg":"bootstrap successfully"} {"level":"info","ts":"2026-03-30T16:47:55.312395Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"a8d4e8c653fa8dce switched to configuration voters=()"} {"level":"info","ts":"2026-03-30T16:47:55.312466Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"a8d4e8c653fa8dce became follower at term 0"} {"level":"info","ts":"2026-03-30T16:47:55.312493Z","logger":"raft","caller":"v3@v3.6.0/raft.go:493","msg":"newRaft a8d4e8c653fa8dce [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"warn","ts":"2026-03-30T16:47:55.334194Z","caller":"auth/store.go:1135","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-30T16:47:55.336364Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-30T16:47:55.345500Z","caller":"storage/quota.go:93","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-30T16:47:55.346175Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.346230Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333","remote-peer-urls":["https://199.19.213.3:2380"]} {"level":"info","ts":"2026-03-30T16:47:55.346282Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.346319Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.346521Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.346621Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.347577Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.347634Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333","remote-peer-urls":["https://199.19.213.3:2380"]} {"level":"info","ts":"2026-03-30T16:47:55.347763Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.347688Z","caller":"etcdserver/server.go:598","msg":"starting etcd server","local-member-id":"a8d4e8c653fa8dce","local-server-version":"3.6.4","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-30T16:47:55.348363Z","caller":"etcdserver/server.go:499","msg":"starting initial election tick advance","election-ticks":10} {"level":"info","ts":"2026-03-30T16:47:55.348485Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-30T16:47:55.348541Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-30T16:47:55.348548Z","caller":"embed/etcd.go:766","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-30T16:47:55.348621Z","caller":"embed/etcd.go:640","msg":"serving peer traffic","address":"199.204.45.227:2380"} {"level":"info","ts":"2026-03-30T16:47:55.348648Z","caller":"embed/etcd.go:611","msg":"cmux::serve","address":"199.204.45.227:2380"} {"level":"info","ts":"2026-03-30T16:47:55.348881Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"warn","ts":"2026-03-30T16:47:55.349434Z","caller":"etcdserver/server.go:2304","msg":"Failed to detect schema version","error":"missing confstate information"} {"level":"info","ts":"2026-03-30T16:47:55.349735Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"a8d4e8c653fa8dce","to":"3cf904ec39d3e333","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-30T16:47:55.349766Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.349817Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.350207Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"a8d4e8c653fa8dce","to":"3cf904ec39d3e333","stream-type":"stream Message"} {"level":"info","ts":"2026-03-30T16:47:55.350273Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.350783Z","caller":"embed/etcd.go:890","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-30T16:47:55.350883Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"a8d4e8c653fa8dce","initial-advertise-peer-urls":["https://199.204.45.227:2380"],"listen-peer-urls":["https://199.204.45.227:2380"],"advertise-client-urls":["https://199.204.45.227:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.227:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-30T16:47:55.352025Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.352465Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"a8d4e8c653fa8dce [term: 0] received a MsgHeartbeat message with higher term from 3cf904ec39d3e333 [term: 2]"} {"level":"info","ts":"2026-03-30T16:47:55.352500Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"a8d4e8c653fa8dce became follower at term 2"} {"level":"info","ts":"2026-03-30T16:47:55.352513Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: a8d4e8c653fa8dce elected leader 3cf904ec39d3e333 at term 2"} {"level":"info","ts":"2026-03-30T16:47:55.357220Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.360540Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.378015Z","caller":"rafthttp/http.go:258","msg":"receiving database snapshot","local-member-id":"a8d4e8c653fa8dce","remote-snapshot-sender-id":"3cf904ec39d3e333","incoming-snapshot-index":542,"incoming-snapshot-message-size-bytes":8193,"incoming-snapshot-message-size":"8.2 kB"} {"level":"info","ts":"2026-03-30T16:47:55.405665Z","caller":"snap/db.go:65","msg":"saved database snapshot to disk","path":"/var/lib/etcd/member/snap/000000000000021e.snap.db","bytes":905216,"size":"905 kB"} {"level":"info","ts":"2026-03-30T16:47:55.405741Z","caller":"rafthttp/http.go:287","msg":"received and saved database snapshot","local-member-id":"a8d4e8c653fa8dce","remote-snapshot-sender-id":"3cf904ec39d3e333","incoming-snapshot-index":542,"incoming-snapshot-size-bytes":905216,"incoming-snapshot-size":"905 kB","download-took":"27.806ms"} {"level":"info","ts":"2026-03-30T16:47:55.406118Z","logger":"raft","caller":"v3@v3.6.0/log.go:465","msg":"log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 542, term: 2]"} {"level":"info","ts":"2026-03-30T16:47:55.406244Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"a8d4e8c653fa8dce switched to configuration voters=(4393548324105610035) learners=(12165604431944650190)"} {"level":"info","ts":"2026-03-30T16:47:55.406279Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1935","msg":"a8d4e8c653fa8dce [commit: 542, lastindex: 542, lastterm: 2] restored snapshot [index: 542, term: 2]"} {"level":"info","ts":"2026-03-30T16:47:55.406296Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1844","msg":"a8d4e8c653fa8dce [commit: 542] restored snapshot [index: 542, term: 2]"} {"level":"info","ts":"2026-03-30T16:47:55.406395Z","caller":"etcdserver/server.go:1007","msg":"applying snapshot","current-snapshot-index":0,"current-applied-index":0,"incoming-leader-snapshot-index":542,"incoming-leader-snapshot-term":2} {"level":"info","ts":"2026-03-30T16:47:55.416576Z","caller":"etcdserver/raft.go:276","msg":"applied incoming Raft snapshot","snapshot-index":542} {"level":"info","ts":"2026-03-30T16:47:55.416869Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc00011a498}"} {"level":"info","ts":"2026-03-30T16:47:55.417368Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-30T16:47:55.417473Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"657.961µs"} {"level":"info","ts":"2026-03-30T16:47:55.417536Z","caller":"etcdserver/server.go:1055","msg":"restoring lease store"} {"level":"info","ts":"2026-03-30T16:47:55.427280Z","caller":"etcdserver/server.go:1059","msg":"restored lease store"} {"level":"info","ts":"2026-03-30T16:47:55.427332Z","caller":"etcdserver/server.go:1062","msg":"restoring mvcc store"} {"level":"info","ts":"2026-03-30T16:47:55.431657Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":513} {"level":"info","ts":"2026-03-30T16:47:55.431708Z","caller":"etcdserver/server.go:1070","msg":"restored mvcc store","consistent-index":542} {"level":"info","ts":"2026-03-30T16:47:55.431726Z","caller":"etcdserver/server.go:1090","msg":"restoring alarm store"} {"level":"info","ts":"2026-03-30T16:47:55.431823Z","caller":"etcdserver/server.go:1078","msg":"closing old backend file"} {"level":"info","ts":"2026-03-30T16:47:55.433188Z","caller":"etcdserver/server.go:1096","msg":"restored alarm store"} {"level":"info","ts":"2026-03-30T16:47:55.433210Z","caller":"etcdserver/server.go:1099","msg":"restoring auth store"} {"level":"info","ts":"2026-03-30T16:47:55.433253Z","caller":"etcdserver/server.go:1103","msg":"restored auth store"} {"level":"info","ts":"2026-03-30T16:47:55.433261Z","caller":"etcdserver/server.go:1106","msg":"restoring v2 store"} {"level":"info","ts":"2026-03-30T16:47:55.434007Z","caller":"etcdserver/server.go:1115","msg":"restored v2 store"} {"level":"info","ts":"2026-03-30T16:47:55.434041Z","caller":"etcdserver/server.go:1119","msg":"restoring cluster configuration"} {"level":"info","ts":"2026-03-30T16:47:55.434088Z","caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.6"} {"level":"info","ts":"2026-03-30T16:47:55.434103Z","caller":"membership/cluster.go:297","msg":"recovered/added member from store","cluster-id":"7b1ae51097bedd0e","local-member-id":"a8d4e8c653fa8dce","recovered-remote-peer-id":"3cf904ec39d3e333","recovered-remote-peer-urls":["https://199.19.213.3:2380"],"recovered-remote-peer-is-learner":false} {"level":"info","ts":"2026-03-30T16:47:55.434115Z","caller":"membership/cluster.go:297","msg":"recovered/added member from store","cluster-id":"7b1ae51097bedd0e","local-member-id":"a8d4e8c653fa8dce","recovered-remote-peer-id":"a8d4e8c653fa8dce","recovered-remote-peer-urls":["https://199.204.45.227:2380"],"recovered-remote-peer-is-learner":true} {"level":"info","ts":"2026-03-30T16:47:55.434123Z","caller":"membership/cluster.go:307","msg":"set cluster version from store","cluster-version":"3.6"} {"level":"info","ts":"2026-03-30T16:47:55.434130Z","caller":"etcdserver/server.go:1123","msg":"restored cluster configuration"} {"level":"info","ts":"2026-03-30T16:47:55.434138Z","caller":"etcdserver/server.go:1124","msg":"removing old peers from network"} {"level":"info","ts":"2026-03-30T16:47:55.434148Z","caller":"rafthttp/peer.go:316","msg":"stopping remote peer","remote-peer-id":"3cf904ec39d3e333"} {"level":"warn","ts":"2026-03-30T16:47:55.434222Z","caller":"rafthttp/stream.go:285","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434245Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"3cf904ec39d3e333"} {"level":"warn","ts":"2026-03-30T16:47:55.434282Z","caller":"rafthttp/stream.go:285","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434296Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434346Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"warn","ts":"2026-03-30T16:47:55.434487Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333","error":"context canceled"} {"level":"warn","ts":"2026-03-30T16:47:55.434511Z","caller":"rafthttp/peer_status.go:66","msg":"peer became inactive (message send to peer failed)","peer-id":"3cf904ec39d3e333","error":"failed to read 3cf904ec39d3e333 on stream MsgApp v2 (context canceled)"} {"level":"info","ts":"2026-03-30T16:47:55.434529Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"warn","ts":"2026-03-30T16:47:55.434590Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333","error":"context canceled"} {"level":"info","ts":"2026-03-30T16:47:55.434575Z","caller":"etcdserver/server.go:1080","msg":"closed old backend file"} {"level":"info","ts":"2026-03-30T16:47:55.434610Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434636Z","caller":"rafthttp/peer.go:321","msg":"stopped remote peer","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434673Z","caller":"rafthttp/transport.go:354","msg":"removed remote peer","local-member-id":"a8d4e8c653fa8dce","removed-remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434692Z","caller":"etcdserver/server.go:1129","msg":"removed old peers from network"} {"level":"info","ts":"2026-03-30T16:47:55.434714Z","caller":"etcdserver/server.go:1130","msg":"adding peers from new cluster configuration"} {"level":"info","ts":"2026-03-30T16:47:55.434745Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434780Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434908Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434938Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.434969Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333","remote-peer-urls":["https://199.19.213.3:2380"]} {"level":"info","ts":"2026-03-30T16:47:55.434990Z","caller":"etcdserver/server.go:1139","msg":"added peers from new cluster configuration"} {"level":"info","ts":"2026-03-30T16:47:55.435001Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.435061Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.435011Z","caller":"etcdserver/server.go:1015","msg":"applied snapshot","current-snapshot-index":542,"current-applied-index":542,"incoming-leader-snapshot-index":542,"incoming-leader-snapshot-term":2} {"level":"info","ts":"2026-03-30T16:47:55.435123Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.443452Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"a8d4e8c653fa8dce","to":"3cf904ec39d3e333","stream-type":"stream Message"} {"level":"info","ts":"2026-03-30T16:47:55.443494Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.443924Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.443756Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.444635Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.445873Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"a8d4e8c653fa8dce","to":"3cf904ec39d3e333","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-30T16:47:55.445889Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"3cf904ec39d3e333"} {"level":"info","ts":"2026-03-30T16:47:55.924048Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"a8d4e8c653fa8dce switched to configuration voters=(4393548324105610035 12165604431944650190)"} {"level":"info","ts":"2026-03-30T16:47:55.924260Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"7b1ae51097bedd0e","local-member-id":"a8d4e8c653fa8dce","promoted-member-id":"a8d4e8c653fa8dce"} {"level":"warn","ts":"2026-03-30T16:48:02.349038Z","caller":"etcdserver/server.go:1814","msg":"failed to publish local member to cluster through raft","local-member-id":"a8d4e8c653fa8dce","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.227:2379]}","publish-timeout":"7s","error":"etcdserver: request timed out, possibly due to connection lost"} {"level":"info","ts":"2026-03-30T16:48:02.867938Z","caller":"etcdserver/server.go:1804","msg":"published local member to cluster through raft","local-member-id":"a8d4e8c653fa8dce","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.227:2379]}","cluster-id":"7b1ae51097bedd0e","publish-timeout":"7s"} {"level":"info","ts":"2026-03-30T16:48:02.867968Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-30T16:48:02.868043Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-30T16:48:02.868275Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-30T16:48:02.868306Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-03-30T16:48:02.869140Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"warn","ts":"2026-03-30T16:48:02.870075Z","caller":"v3rpc/grpc.go:52","msg":"etcdserver: failed to register grpc metrics","error":"duplicate metrics collector registration attempted"} {"level":"info","ts":"2026-03-30T16:48:02.870181Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-30T16:48:02.871948Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.204.45.227:2379"} {"level":"info","ts":"2026-03-30T16:48:02.873057Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"warn","ts":"2026-03-30T16:48:02.875221Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:57734","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.875433Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:57718","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.877095Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:35288","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.877276Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:35310","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.878361Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:35262","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.879753Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:57700","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.880707Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:35300","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.880889Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:35320","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.881495Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:35274","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.881486Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:35270","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.882244Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:51660","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.882506Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:51648","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.885346Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.227:35330","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.889617Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55082","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.906546Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55104","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.922600Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55138","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.932121Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55152","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.940432Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55184","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.949105Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55202","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.957396Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55214","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.965712Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55228","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.976013Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55246","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.985747Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55252","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:02.994564Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55272","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.003629Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55288","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.011315Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55328","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.024112Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55344","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.032447Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55358","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.043351Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55380","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.061496Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55420","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.063937Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"123.815713ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:41 size:3525"} {"level":"info","ts":"2026-03-30T16:48:03.064012Z","caller":"traceutil/trace.go:172","msg":"trace[1087449804] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:41; response_revision:528; }","duration":"123.955735ms","start":"2026-03-30T16:48:02.940041Z","end":"2026-03-30T16:48:03.063996Z","steps":["trace[1087449804] 'agreement among raft nodes before linearized reading' (duration: 50.933916ms)","trace[1087449804] 'range keys from in-memory index tree' (duration: 72.655023ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:03.064281Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"114.837662ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourcequotas/\" range_end:\"/registry/resourcequotas0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.064326Z","caller":"traceutil/trace.go:172","msg":"trace[793325715] range","detail":"{range_begin:/registry/resourcequotas/; range_end:/registry/resourcequotas0; response_count:0; response_revision:530; }","duration":"114.891212ms","start":"2026-03-30T16:48:02.949425Z","end":"2026-03-30T16:48:03.064316Z","steps":["trace[793325715] 'agreement among raft nodes before linearized reading' (duration: 114.820251ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.064264Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"114.647798ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourcequotas\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:48:03.064852Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"108.334383ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.064877Z","caller":"traceutil/trace.go:172","msg":"trace[278698389] range","detail":"{range_begin:/registry/secrets; range_end:; response_count:0; response_revision:530; }","duration":"108.360003ms","start":"2026-03-30T16:48:02.956510Z","end":"2026-03-30T16:48:03.064870Z","steps":["trace[278698389] 'agreement among raft nodes before linearized reading' (duration: 108.299222ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.065062Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"108.555726ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets/\" range_end:\"/registry/secrets0\" limit:10000 ","response":"range_response_count:5 size:13879"} {"level":"info","ts":"2026-03-30T16:48:03.065132Z","caller":"traceutil/trace.go:172","msg":"trace[634158056] range","detail":"{range_begin:/registry/secrets/; range_end:/registry/secrets0; response_count:5; response_revision:530; }","duration":"108.628007ms","start":"2026-03-30T16:48:02.956488Z","end":"2026-03-30T16:48:03.065116Z","steps":["trace[634158056] 'agreement among raft nodes before linearized reading' (duration: 108.448495ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:03.064458Z","caller":"traceutil/trace.go:172","msg":"trace[362658934] range","detail":"{range_begin:/registry/resourcequotas; range_end:; response_count:0; response_revision:530; }","duration":"114.853761ms","start":"2026-03-30T16:48:02.949590Z","end":"2026-03-30T16:48:03.064444Z","steps":["trace[362658934] 'agreement among raft nodes before linearized reading' (duration: 114.618218ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.072145Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55444","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.105859Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55456","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.115509Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55470","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.123806Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55492","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.137302Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55502","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.146484Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55508","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.158311Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55516","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.169809Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55528","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.180342Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55554","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.190667Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55576","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.212190Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55598","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.221571Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55628","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.248881Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55638","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.256991Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55656","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.269964Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55680","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.277828Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55690","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.321599Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55704","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.332225Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55734","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.352009Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55748","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T16:48:03.359518Z","caller":"traceutil/trace.go:172","msg":"trace[730136665] linearizableReadLoop","detail":"{readStateIndex:564; appliedIndex:564; }","duration":"245.354201ms","start":"2026-03-30T16:48:03.114139Z","end":"2026-03-30T16:48:03.359494Z","steps":["trace[730136665] 'read index received' (duration: 245.348421ms)","trace[730136665] 'applied index is now lower than readState.Index' (duration: 4.75µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:03.359661Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"245.508794ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.359713Z","caller":"traceutil/trace.go:172","msg":"trace[1420764125] range","detail":"{range_begin:/registry/horizontalpodautoscalers; range_end:; response_count:0; response_revision:531; }","duration":"245.572325ms","start":"2026-03-30T16:48:03.114130Z","end":"2026-03-30T16:48:03.359702Z","steps":["trace[1420764125] 'agreement among raft nodes before linearized reading' (duration: 245.480063ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.360616Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55764","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.361307Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"149.498093ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ipaddresses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.361391Z","caller":"traceutil/trace.go:172","msg":"trace[248294978] range","detail":"{range_begin:/registry/ipaddresses; range_end:; response_count:0; response_revision:531; }","duration":"149.584845ms","start":"2026-03-30T16:48:03.211787Z","end":"2026-03-30T16:48:03.361372Z","steps":["trace[248294978] 'agreement among raft nodes before linearized reading' (duration: 149.407122ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.361410Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"149.622435ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ipaddresses/\" range_end:\"/registry/ipaddresses0\" limit:10000 ","response":"range_response_count:2 size:1169"} {"level":"info","ts":"2026-03-30T16:48:03.361470Z","caller":"traceutil/trace.go:172","msg":"trace[1264615141] range","detail":"{range_begin:/registry/ipaddresses/; range_end:/registry/ipaddresses0; response_count:2; response_revision:531; }","duration":"149.683466ms","start":"2026-03-30T16:48:03.211772Z","end":"2026-03-30T16:48:03.361455Z","steps":["trace[1264615141] 'agreement among raft nodes before linearized reading' (duration: 149.489733ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.361731Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"159.637573ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingressclasses/\" range_end:\"/registry/ingressclasses0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.361794Z","caller":"traceutil/trace.go:172","msg":"trace[372781254] range","detail":"{range_begin:/registry/ingressclasses/; range_end:/registry/ingressclasses0; response_count:0; response_revision:531; }","duration":"159.752864ms","start":"2026-03-30T16:48:03.202021Z","end":"2026-03-30T16:48:03.361774Z","steps":["trace[372781254] 'agreement among raft nodes before linearized reading' (duration: 159.614872ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.361862Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"130.511577ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/runtimeclasses/\" range_end:\"/registry/runtimeclasses0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.362174Z","caller":"traceutil/trace.go:172","msg":"trace[158234627] range","detail":"{range_begin:/registry/runtimeclasses/; range_end:/registry/runtimeclasses0; response_count:0; response_revision:531; }","duration":"130.831773ms","start":"2026-03-30T16:48:03.231325Z","end":"2026-03-30T16:48:03.362157Z","steps":["trace[158234627] 'agreement among raft nodes before linearized reading' (duration: 130.423086ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.362394Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"217.259152ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/certificatesigningrequests/\" range_end:\"/registry/certificatesigningrequests0\" limit:10000 ","response":"range_response_count:1 size:2419"} {"level":"info","ts":"2026-03-30T16:48:03.362454Z","caller":"traceutil/trace.go:172","msg":"trace[857550587] range","detail":"{range_begin:/registry/certificatesigningrequests/; range_end:/registry/certificatesigningrequests0; response_count:1; response_revision:531; }","duration":"217.314573ms","start":"2026-03-30T16:48:03.145118Z","end":"2026-03-30T16:48:03.362433Z","steps":["trace[857550587] 'agreement among raft nodes before linearized reading' (duration: 217.177181ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.362676Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"217.524847ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/certificatesigningrequests\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.362841Z","caller":"traceutil/trace.go:172","msg":"trace[1620108970] range","detail":"{range_begin:/registry/certificatesigningrequests; range_end:; response_count:0; response_revision:531; }","duration":"217.647539ms","start":"2026-03-30T16:48:03.145130Z","end":"2026-03-30T16:48:03.362778Z","steps":["trace[1620108970] 'agreement among raft nodes before linearized reading' (duration: 217.1295ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.362845Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"114.479558ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/roles/\" range_end:\"/registry/roles0\" limit:10000 ","response":"range_response_count:12 size:6572"} {"level":"info","ts":"2026-03-30T16:48:03.362904Z","caller":"traceutil/trace.go:172","msg":"trace[2100081435] range","detail":"{range_begin:/registry/roles/; range_end:/registry/roles0; response_count:12; response_revision:531; }","duration":"114.539839ms","start":"2026-03-30T16:48:03.248351Z","end":"2026-03-30T16:48:03.362891Z","steps":["trace[2100081435] 'agreement among raft nodes before linearized reading' (duration: 114.354986ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.363144Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"105.861136ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.363193Z","caller":"traceutil/trace.go:172","msg":"trace[1494051160] range","detail":"{range_begin:/registry/rolebindings; range_end:; response_count:0; response_revision:531; }","duration":"105.915006ms","start":"2026-03-30T16:48:03.257265Z","end":"2026-03-30T16:48:03.363180Z","steps":["trace[1494051160] 'agreement among raft nodes before linearized reading' (duration: 105.830244ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.363200Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"123.311157ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/poddisruptionbudgets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.363241Z","caller":"traceutil/trace.go:172","msg":"trace[858594396] range","detail":"{range_begin:/registry/poddisruptionbudgets; range_end:; response_count:0; response_revision:531; }","duration":"123.354387ms","start":"2026-03-30T16:48:03.239875Z","end":"2026-03-30T16:48:03.363230Z","steps":["trace[858594396] 'agreement among raft nodes before linearized reading' (duration: 123.286606ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.363258Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"173.009575ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingress/\" range_end:\"/registry/ingress0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.363343Z","caller":"traceutil/trace.go:172","msg":"trace[1742750304] range","detail":"{range_begin:/registry/ingress/; range_end:/registry/ingress0; response_count:0; response_revision:531; }","duration":"173.099807ms","start":"2026-03-30T16:48:03.190228Z","end":"2026-03-30T16:48:03.363327Z","steps":["trace[1742750304] 'agreement among raft nodes before linearized reading' (duration: 172.948714ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.363427Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"131.718626ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/runtimeclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.363463Z","caller":"traceutil/trace.go:172","msg":"trace[829091798] range","detail":"{range_begin:/registry/runtimeclasses; range_end:; response_count:0; response_revision:531; }","duration":"131.750737ms","start":"2026-03-30T16:48:03.231700Z","end":"2026-03-30T16:48:03.363451Z","steps":["trace[829091798] 'agreement among raft nodes before linearized reading' (duration: 131.700586ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.363501Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"106.705919ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rolebindings/\" range_end:\"/registry/rolebindings0\" limit:10000 ","response":"range_response_count:12 size:7968"} {"level":"warn","ts":"2026-03-30T16:48:03.362264Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"160.0992ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingressclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.363545Z","caller":"traceutil/trace.go:172","msg":"trace[1796811351] range","detail":"{range_begin:/registry/rolebindings/; range_end:/registry/rolebindings0; response_count:12; response_revision:531; }","duration":"106.75144ms","start":"2026-03-30T16:48:03.256782Z","end":"2026-03-30T16:48:03.363533Z","steps":["trace[1796811351] 'agreement among raft nodes before linearized reading' (duration: 106.585517ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:03.363567Z","caller":"traceutil/trace.go:172","msg":"trace[1671012659] range","detail":"{range_begin:/registry/ingressclasses; range_end:; response_count:0; response_revision:531; }","duration":"161.534825ms","start":"2026-03-30T16:48:03.202020Z","end":"2026-03-30T16:48:03.363555Z","steps":["trace[1671012659] 'agreement among raft nodes before linearized reading' (duration: 160.0715ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.363734Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"173.539484ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingress\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.363815Z","caller":"traceutil/trace.go:172","msg":"trace[593016171] range","detail":"{range_begin:/registry/ingress; range_end:; response_count:0; response_revision:531; }","duration":"173.619646ms","start":"2026-03-30T16:48:03.190182Z","end":"2026-03-30T16:48:03.363802Z","steps":["trace[593016171] 'agreement among raft nodes before linearized reading' (duration: 173.516164ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.364181Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"184.006519ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/networkpolicies\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.364223Z","caller":"traceutil/trace.go:172","msg":"trace[750505214] range","detail":"{range_begin:/registry/networkpolicies; range_end:; response_count:0; response_revision:531; }","duration":"184.09644ms","start":"2026-03-30T16:48:03.180115Z","end":"2026-03-30T16:48:03.364211Z","steps":["trace[750505214] 'agreement among raft nodes before linearized reading' (duration: 183.970318ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.364416Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"249.866787ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:48:03.364437Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"207.408379ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.364485Z","caller":"traceutil/trace.go:172","msg":"trace[501827884] range","detail":"{range_begin:/registry/horizontalpodautoscalers/; range_end:/registry/horizontalpodautoscalers0; response_count:0; response_revision:531; }","duration":"249.957868ms","start":"2026-03-30T16:48:03.114505Z","end":"2026-03-30T16:48:03.364463Z","steps":["trace[501827884] 'agreement among raft nodes before linearized reading' (duration: 249.845907ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:03.364493Z","caller":"traceutil/trace.go:172","msg":"trace[1650276074] range","detail":"{range_begin:/registry/leases; range_end:; response_count:0; response_revision:531; }","duration":"207.47058ms","start":"2026-03-30T16:48:03.157010Z","end":"2026-03-30T16:48:03.364480Z","steps":["trace[1650276074] 'agreement among raft nodes before linearized reading' (duration: 207.373448ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.364643Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"226.924844ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cronjobs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.364670Z","caller":"traceutil/trace.go:172","msg":"trace[1022973818] range","detail":"{range_begin:/registry/cronjobs; range_end:; response_count:0; response_revision:531; }","duration":"226.951605ms","start":"2026-03-30T16:48:03.137710Z","end":"2026-03-30T16:48:03.364661Z","steps":["trace[1022973818] 'agreement among raft nodes before linearized reading' (duration: 226.907034ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.364690Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"184.597258ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/networkpolicies/\" range_end:\"/registry/networkpolicies0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.364729Z","caller":"traceutil/trace.go:172","msg":"trace[844373971] range","detail":"{range_begin:/registry/networkpolicies/; range_end:/registry/networkpolicies0; response_count:0; response_revision:531; }","duration":"184.63978ms","start":"2026-03-30T16:48:03.180078Z","end":"2026-03-30T16:48:03.364718Z","steps":["trace[844373971] 'agreement among raft nodes before linearized reading' (duration: 184.575898ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.364780Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"207.766985ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/\" range_end:\"/registry/leases0\" limit:10000 ","response":"range_response_count:6 size:3080"} {"level":"info","ts":"2026-03-30T16:48:03.365088Z","caller":"traceutil/trace.go:172","msg":"trace[890757176] range","detail":"{range_begin:/registry/leases/; range_end:/registry/leases0; response_count:6; response_revision:531; }","duration":"208.08344ms","start":"2026-03-30T16:48:03.156987Z","end":"2026-03-30T16:48:03.365071Z","steps":["trace[890757176] 'agreement among raft nodes before linearized reading' (duration: 207.584762ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.361332Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"112.940663ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/roles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:48:03.364853Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"241.494557ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/jobs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:48:03.365430Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"195.851436ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/\" range_end:\"/registry/endpointslices0\" limit:10000 ","response":"range_response_count:2 size:1527"} {"level":"info","ts":"2026-03-30T16:48:03.365434Z","caller":"traceutil/trace.go:172","msg":"trace[1980708729] range","detail":"{range_begin:/registry/jobs; range_end:; response_count:0; response_revision:531; }","duration":"242.063756ms","start":"2026-03-30T16:48:03.123353Z","end":"2026-03-30T16:48:03.365417Z","steps":["trace[1980708729] 'agreement among raft nodes before linearized reading' (duration: 241.470336ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:03.365471Z","caller":"traceutil/trace.go:172","msg":"trace[108356606] range","detail":"{range_begin:/registry/endpointslices/; range_end:/registry/endpointslices0; response_count:2; response_revision:531; }","duration":"195.895747ms","start":"2026-03-30T16:48:03.169563Z","end":"2026-03-30T16:48:03.365459Z","steps":["trace[108356606] 'agreement among raft nodes before linearized reading' (duration: 195.752244ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.365516Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"145.412345ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/servicecidrs/\" range_end:\"/registry/servicecidrs0\" limit:10000 ","response":"range_response_count:1 size:1006"} {"level":"info","ts":"2026-03-30T16:48:03.365567Z","caller":"traceutil/trace.go:172","msg":"trace[1361198562] range","detail":"{range_begin:/registry/servicecidrs/; range_end:/registry/servicecidrs0; response_count:1; response_revision:531; }","duration":"145.465146ms","start":"2026-03-30T16:48:03.220088Z","end":"2026-03-30T16:48:03.365553Z","steps":["trace[1361198562] 'agreement among raft nodes before linearized reading' (duration: 145.335624ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.363378Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"123.534471ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/poddisruptionbudgets/\" range_end:\"/registry/poddisruptionbudgets0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.365679Z","caller":"traceutil/trace.go:172","msg":"trace[1514502848] range","detail":"{range_begin:/registry/poddisruptionbudgets/; range_end:/registry/poddisruptionbudgets0; response_count:0; response_revision:531; }","duration":"125.833869ms","start":"2026-03-30T16:48:03.239830Z","end":"2026-03-30T16:48:03.365664Z","steps":["trace[1514502848] 'agreement among raft nodes before linearized reading' (duration: 123.512121ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.365686Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"196.07287ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:48:03.364886Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"241.555258ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/jobs/\" range_end:\"/registry/jobs0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.365868Z","caller":"traceutil/trace.go:172","msg":"trace[1783462400] range","detail":"{range_begin:/registry/endpointslices; range_end:; response_count:0; response_revision:531; }","duration":"196.252193ms","start":"2026-03-30T16:48:03.169602Z","end":"2026-03-30T16:48:03.365854Z","steps":["trace[1783462400] 'agreement among raft nodes before linearized reading' (duration: 196.04887ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:03.365875Z","caller":"traceutil/trace.go:172","msg":"trace[1364742179] range","detail":"{range_begin:/registry/roles; range_end:; response_count:0; response_revision:531; }","duration":"117.492188ms","start":"2026-03-30T16:48:03.248366Z","end":"2026-03-30T16:48:03.365858Z","steps":["trace[1364742179] 'agreement among raft nodes before linearized reading' (duration: 112.903312ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:03.365901Z","caller":"traceutil/trace.go:172","msg":"trace[506817618] range","detail":"{range_begin:/registry/jobs/; range_end:/registry/jobs0; response_count:0; response_revision:531; }","duration":"242.559175ms","start":"2026-03-30T16:48:03.123325Z","end":"2026-03-30T16:48:03.365884Z","steps":["trace[506817618] 'agreement among raft nodes before linearized reading' (duration: 241.542408ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.364807Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"227.226599ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cronjobs/\" range_end:\"/registry/cronjobs0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.366009Z","caller":"traceutil/trace.go:172","msg":"trace[744395555] range","detail":"{range_begin:/registry/cronjobs/; range_end:/registry/cronjobs0; response_count:0; response_revision:531; }","duration":"228.424849ms","start":"2026-03-30T16:48:03.137568Z","end":"2026-03-30T16:48:03.365993Z","steps":["trace[744395555] 'agreement among raft nodes before linearized reading' (duration: 227.212949ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.366246Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"146.163348ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/servicecidrs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:03.366337Z","caller":"traceutil/trace.go:172","msg":"trace[324404293] range","detail":"{range_begin:/registry/servicecidrs; range_end:; response_count:0; response_revision:531; }","duration":"146.2532ms","start":"2026-03-30T16:48:03.220072Z","end":"2026-03-30T16:48:03.366325Z","steps":["trace[324404293] 'agreement among raft nodes before linearized reading' (duration: 145.140771ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:03.379638Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55782","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.386121Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55800","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.393926Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55810","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.402519Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55834","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.410750Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55840","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.418543Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55862","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.428720Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55872","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.438324Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55888","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.449846Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55904","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.457726Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55912","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.465801Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55928","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.474974Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55952","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.485161Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55982","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.492967Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:55984","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.522351Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:56008","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.534715Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:56020","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.546337Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:56048","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:03.604751Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:56056","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T16:48:04.506085Z","caller":"traceutil/trace.go:172","msg":"trace[82312408] linearizableReadLoop","detail":"{readStateIndex:564; appliedIndex:565; }","duration":"345.078887ms","start":"2026-03-30T16:48:04.160983Z","end":"2026-03-30T16:48:04.506062Z","steps":["trace[82312408] 'read index received' (duration: 345.070747ms)","trace[82312408] 'applied index is now lower than readState.Index' (duration: 6.91µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:04.506213Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"345.212641ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-30T16:48:04.506247Z","caller":"traceutil/trace.go:172","msg":"trace[1881423413] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:532; }","duration":"345.260231ms","start":"2026-03-30T16:48:04.160977Z","end":"2026-03-30T16:48:04.506238Z","steps":["trace[1881423413] 'agreement among raft nodes before linearized reading' (duration: 345.17518ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:04.506277Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:04.160958Z","time spent":"345.311192ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"info","ts":"2026-03-30T16:48:04.507067Z","caller":"traceutil/trace.go:172","msg":"trace[641887743] transaction","detail":"{read_only:false; response_revision:533; number_of_response:1; }","duration":"377.758208ms","start":"2026-03-30T16:48:04.129284Z","end":"2026-03-30T16:48:04.507043Z","steps":["trace[641887743] 'process raft request' (duration: 377.621476ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:04.507744Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:04.129265Z","time spent":"377.88454ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":676,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T16:48:07.278626Z","caller":"traceutil/trace.go:172","msg":"trace[2128194460] linearizableReadLoop","detail":"{readStateIndex:578; appliedIndex:579; }","duration":"148.771258ms","start":"2026-03-30T16:48:07.129834Z","end":"2026-03-30T16:48:07.278605Z","steps":["trace[2128194460] 'read index received' (duration: 148.766418ms)","trace[2128194460] 'applied index is now lower than readState.Index' (duration: 4.07µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:07.279019Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"149.164784ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:48:07.279069Z","caller":"traceutil/trace.go:172","msg":"trace[1517104560] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:544; }","duration":"149.229945ms","start":"2026-03-30T16:48:07.129829Z","end":"2026-03-30T16:48:07.279059Z","steps":["trace[1517104560] 'agreement among raft nodes before linearized reading' (duration: 148.87738ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:07.279785Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"101.44754ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:07.279816Z","caller":"traceutil/trace.go:172","msg":"trace[1708288917] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:544; }","duration":"101.48053ms","start":"2026-03-30T16:48:07.178329Z","end":"2026-03-30T16:48:07.279809Z","steps":["trace[1708288917] 'agreement among raft nodes before linearized reading' (duration: 101.436129ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:09.700238Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.507035173s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:09.700306Z","caller":"traceutil/trace.go:172","msg":"trace[751425777] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:544; }","duration":"1.507110384s","start":"2026-03-30T16:48:08.193182Z","end":"2026-03-30T16:48:09.700292Z","steps":["trace[751425777] 'range keys from in-memory index tree' (duration: 1.506997651s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:09.700502Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.467758318s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:48:09.700620Z","caller":"traceutil/trace.go:172","msg":"trace[467853324] linearizableReadLoop","detail":"{readStateIndex:580; appliedIndex:579; }","duration":"1.882270011s","start":"2026-03-30T16:48:07.818333Z","end":"2026-03-30T16:48:09.700603Z","steps":["trace[467853324] 'read index received' (duration: 415.040601ms)","trace[467853324] 'applied index is now lower than readState.Index' (duration: 1.46722844s)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:09.700778Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.882416633s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T16:48:09.700810Z","caller":"traceutil/trace.go:172","msg":"trace[784106363] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:546; }","duration":"1.882474224s","start":"2026-03-30T16:48:07.818329Z","end":"2026-03-30T16:48:09.700803Z","steps":["trace[784106363] 'agreement among raft nodes before linearized reading' (duration: 1.882387773s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:09.700843Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:07.818311Z","time spent":"1.882522434s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":506,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:09.718757Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"326.325765ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T16:48:09.719341Z","caller":"traceutil/trace.go:172","msg":"trace[1980259520] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:548; }","duration":"326.920024ms","start":"2026-03-30T16:48:09.392405Z","end":"2026-03-30T16:48:09.719325Z","steps":["trace[1980259520] 'agreement among raft nodes before linearized reading' (duration: 326.185553ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:09.719410Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:09.392384Z","time spent":"327.013947ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:11.642407Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"106.980326ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:11.642504Z","caller":"traceutil/trace.go:172","msg":"trace[1707337976] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:551; }","duration":"107.095597ms","start":"2026-03-30T16:48:11.535391Z","end":"2026-03-30T16:48:11.642487Z","steps":["trace[1707337976] 'range keys from in-memory index tree' (duration: 105.550923ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:12.552014Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"253.379983ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:48:12.552335Z","caller":"traceutil/trace.go:172","msg":"trace[113300728] linearizableReadLoop","detail":"{readStateIndex:590; appliedIndex:589; }","duration":"373.748289ms","start":"2026-03-30T16:48:12.178559Z","end":"2026-03-30T16:48:12.552308Z","steps":["trace[113300728] 'read index received' (duration: 120.957047ms)","trace[113300728] 'applied index is now lower than readState.Index' (duration: 252.788712ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:12.552634Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"374.065494ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:12.552959Z","caller":"traceutil/trace.go:172","msg":"trace[1555026328] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:556; }","duration":"374.395359ms","start":"2026-03-30T16:48:12.178542Z","end":"2026-03-30T16:48:12.552937Z","steps":["trace[1555026328] 'agreement among raft nodes before linearized reading' (duration: 374.039674ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:13.766392Z","caller":"traceutil/trace.go:172","msg":"trace[292324365] linearizableReadLoop","detail":"{readStateIndex:593; appliedIndex:594; }","duration":"249.539087ms","start":"2026-03-30T16:48:13.516831Z","end":"2026-03-30T16:48:13.766371Z","steps":["trace[292324365] 'read index received' (duration: 249.525027ms)","trace[292324365] 'applied index is now lower than readState.Index' (duration: 13.16µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:13.766531Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"249.677089ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csinodes/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:13.766807Z","caller":"traceutil/trace.go:172","msg":"trace[1588604215] range","detail":"{range_begin:/registry/csinodes/controller-2; range_end:; response_count:0; response_revision:558; }","duration":"249.72682ms","start":"2026-03-30T16:48:13.516827Z","end":"2026-03-30T16:48:13.766554Z","steps":["trace[1588604215] 'agreement among raft nodes before linearized reading' (duration: 249.643959ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:13.767876Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"233.472446ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:13.767949Z","caller":"traceutil/trace.go:172","msg":"trace[1392964332] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:559; }","duration":"233.545367ms","start":"2026-03-30T16:48:13.534392Z","end":"2026-03-30T16:48:13.767937Z","steps":["trace[1392964332] 'agreement among raft nodes before linearized reading' (duration: 233.422375ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:13.768079Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"200.493279ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T16:48:13.768163Z","caller":"traceutil/trace.go:172","msg":"trace[1121269478] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:559; }","duration":"200.57568ms","start":"2026-03-30T16:48:13.567571Z","end":"2026-03-30T16:48:13.768147Z","steps":["trace[1121269478] 'agreement among raft nodes before linearized reading' (duration: 200.331887ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:15.584308Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"516.249138ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csinodes/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:15.584377Z","caller":"traceutil/trace.go:172","msg":"trace[864370185] range","detail":"{range_begin:/registry/csinodes/controller-2; range_end:; response_count:0; response_revision:562; }","duration":"516.314639ms","start":"2026-03-30T16:48:15.068045Z","end":"2026-03-30T16:48:15.584360Z","steps":["trace[864370185] 'range keys from in-memory index tree' (duration: 514.843507ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:15.584421Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:15.068008Z","time spent":"516.39865ms","remote":"127.0.0.1:55754","response type":"/etcdserverpb.KV/Range","request count":0,"request size":35,"response count":0,"response size":28,"request content":"key:\"/registry/csinodes/controller-2\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:15.584433Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"187.346189ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:15.584572Z","caller":"traceutil/trace.go:172","msg":"trace[1236352686] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:562; }","duration":"187.513641ms","start":"2026-03-30T16:48:15.397030Z","end":"2026-03-30T16:48:15.584544Z","steps":["trace[1236352686] 'range keys from in-memory index tree' (duration: 187.294258ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:15.584157Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"405.89378ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:15.586167Z","caller":"traceutil/trace.go:172","msg":"trace[932924758] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:562; }","duration":"407.914812ms","start":"2026-03-30T16:48:15.178234Z","end":"2026-03-30T16:48:15.586149Z","steps":["trace[932924758] 'range keys from in-memory index tree' (duration: 403.522964ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:15.584854Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"572.771723ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:48:15.586532Z","caller":"traceutil/trace.go:172","msg":"trace[1806910088] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:562; }","duration":"574.474699ms","start":"2026-03-30T16:48:15.012041Z","end":"2026-03-30T16:48:15.586516Z","steps":["trace[1806910088] 'range keys from in-memory index tree' (duration: 570.421807ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:15.586585Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:15.012019Z","time spent":"574.54675ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:18.267297Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228003631,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:48:18.370529Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"191.322719ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:18.370682Z","caller":"traceutil/trace.go:172","msg":"trace[13056084] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:581; }","duration":"191.463852ms","start":"2026-03-30T16:48:18.179168Z","end":"2026-03-30T16:48:18.370632Z","steps":["trace[13056084] 'range keys from in-memory index tree' (duration: 191.230928ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:18.370818Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"429.586388ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:48:18.370931Z","caller":"traceutil/trace.go:172","msg":"trace[2135953463] transaction","detail":"{read_only:false; response_revision:582; number_of_response:1; }","duration":"655.181468ms","start":"2026-03-30T16:48:17.715729Z","end":"2026-03-30T16:48:18.370910Z","steps":["trace[2135953463] 'process raft request' (duration: 225.403837ms)","trace[2135953463] 'compare' (duration: 429.394815ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:18.371002Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:17.715696Z","time spent":"655.279759ms","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5683,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T16:48:18.376308Z","caller":"traceutil/trace.go:172","msg":"trace[1555845015] linearizableReadLoop","detail":"{readStateIndex:622; appliedIndex:623; }","duration":"609.218674ms","start":"2026-03-30T16:48:17.767067Z","end":"2026-03-30T16:48:18.376285Z","steps":["trace[1555845015] 'read index received' (duration: 609.212514ms)","trace[1555845015] 'applied index is now lower than readState.Index' (duration: 5.35µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:18.376580Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"609.502468ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/kube-system/kube-proxy\" limit:1 ","response":"range_response_count:1 size:185"} {"level":"info","ts":"2026-03-30T16:48:18.376631Z","caller":"traceutil/trace.go:172","msg":"trace[1797854079] range","detail":"{range_begin:/registry/serviceaccounts/kube-system/kube-proxy; range_end:; response_count:1; response_revision:584; }","duration":"609.569209ms","start":"2026-03-30T16:48:17.767051Z","end":"2026-03-30T16:48:18.376620Z","steps":["trace[1797854079] 'agreement among raft nodes before linearized reading' (duration: 609.372086ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:18.376689Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:17.767024Z","time spent":"609.644561ms","remote":"127.0.0.1:55430","response type":"/etcdserverpb.KV/Range","request count":0,"request size":52,"response count":1,"response size":208,"request content":"key:\"/registry/serviceaccounts/kube-system/kube-proxy\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:18.378121Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"133.706892ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"warn","ts":"2026-03-30T16:48:18.378181Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"523.577723ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:48:18.378229Z","caller":"traceutil/trace.go:172","msg":"trace[1586902390] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:584; }","duration":"133.825114ms","start":"2026-03-30T16:48:18.244392Z","end":"2026-03-30T16:48:18.378217Z","steps":["trace[1586902390] 'agreement among raft nodes before linearized reading' (duration: 133.623791ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:18.378259Z","caller":"traceutil/trace.go:172","msg":"trace[735226530] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:584; }","duration":"523.655405ms","start":"2026-03-30T16:48:17.854595Z","end":"2026-03-30T16:48:18.378250Z","steps":["trace[735226530] 'agreement among raft nodes before linearized reading' (duration: 523.560303ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:18.378391Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:17.854190Z","time spent":"524.086742ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:21.036274Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228003663,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:48:21.479033Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.084095329s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T16:48:21.537106Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228003663,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:48:21.815241Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"636.07485ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:21.815349Z","caller":"traceutil/trace.go:172","msg":"trace[1111552849] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:601; }","duration":"636.201252ms","start":"2026-03-30T16:48:21.179129Z","end":"2026-03-30T16:48:21.815331Z","steps":["trace[1111552849] 'range keys from in-memory index tree' (duration: 636.021079ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:21.836663Z","caller":"traceutil/trace.go:172","msg":"trace[1117840304] linearizableReadLoop","detail":"{readStateIndex:642; appliedIndex:642; }","duration":"1.30101617s","start":"2026-03-30T16:48:20.535612Z","end":"2026-03-30T16:48:21.836629Z","steps":["trace[1117840304] 'read index received' (duration: 1.300936799s)","trace[1117840304] 'applied index is now lower than readState.Index' (duration: 76.271µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:21.838410Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.302780136s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:21.838475Z","caller":"traceutil/trace.go:172","msg":"trace[387738738] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:602; }","duration":"1.302851247s","start":"2026-03-30T16:48:20.535607Z","end":"2026-03-30T16:48:21.838458Z","steps":["trace[387738738] 'agreement among raft nodes before linearized reading' (duration: 1.301099331s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:21.838517Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:20.535584Z","time spent":"1.302923999s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:48:21.841553Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.239744129s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:48:21.841600Z","caller":"traceutil/trace.go:172","msg":"trace[1356756021] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:603; }","duration":"1.2397922s","start":"2026-03-30T16:48:20.601797Z","end":"2026-03-30T16:48:21.841589Z","steps":["trace[1356756021] 'agreement among raft nodes before linearized reading' (duration: 1.239668658s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:21.841630Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:20.601774Z","time spent":"1.239847721s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:34.785795Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"249.283685ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:34.785894Z","caller":"traceutil/trace.go:172","msg":"trace[1453938656] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:637; }","duration":"249.398807ms","start":"2026-03-30T16:48:34.536476Z","end":"2026-03-30T16:48:34.785875Z","steps":["trace[1453938656] 'range keys from in-memory index tree' (duration: 247.070053ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:35.548988Z","caller":"traceutil/trace.go:172","msg":"trace[1184901767] transaction","detail":"{read_only:false; response_revision:638; number_of_response:1; }","duration":"176.965771ms","start":"2026-03-30T16:48:35.372003Z","end":"2026-03-30T16:48:35.548969Z","steps":["trace[1184901767] 'process raft request' (duration: 176.817319ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:35.945814Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"337.106948ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:48:35.945946Z","caller":"traceutil/trace.go:172","msg":"trace[582028457] transaction","detail":"{read_only:false; response_revision:639; number_of_response:1; }","duration":"386.370794ms","start":"2026-03-30T16:48:35.559553Z","end":"2026-03-30T16:48:35.945924Z","steps":["trace[582028457] 'process raft request' (duration: 48.87296ms)","trace[582028457] 'compare' (duration: 336.899015ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:35.946045Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:35.559539Z","time spent":"386.462285ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:48:35.972267Z","caller":"traceutil/trace.go:172","msg":"trace[1975823783] linearizableReadLoop","detail":"{readStateIndex:687; appliedIndex:687; }","duration":"265.606084ms","start":"2026-03-30T16:48:35.706636Z","end":"2026-03-30T16:48:35.972242Z","steps":["trace[1975823783] 'read index received' (duration: 265.598474ms)","trace[1975823783] 'applied index is now lower than readState.Index' (duration: 5.99µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:35.972433Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"265.763637ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:48:35.972528Z","caller":"traceutil/trace.go:172","msg":"trace[249213447] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:640; }","duration":"265.889718ms","start":"2026-03-30T16:48:35.706619Z","end":"2026-03-30T16:48:35.972509Z","steps":["trace[249213447] 'agreement among raft nodes before linearized reading' (duration: 265.716446ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:35.974963Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"151.74318ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T16:48:35.975099Z","caller":"traceutil/trace.go:172","msg":"trace[6689884] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:640; }","duration":"151.870802ms","start":"2026-03-30T16:48:35.823189Z","end":"2026-03-30T16:48:35.975060Z","steps":["trace[6689884] 'agreement among raft nodes before linearized reading' (duration: 151.593378ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:38.679844Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228003740,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:48:38.829559Z","caller":"traceutil/trace.go:172","msg":"trace[384043886] linearizableReadLoop","detail":"{readStateIndex:694; appliedIndex:695; }","duration":"650.300707ms","start":"2026-03-30T16:48:38.179234Z","end":"2026-03-30T16:48:38.829534Z","steps":["trace[384043886] 'read index received' (duration: 650.290436ms)","trace[384043886] 'applied index is now lower than readState.Index' (duration: 8.531µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:38.829750Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"650.492969ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:38.829815Z","caller":"traceutil/trace.go:172","msg":"trace[631160304] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:647; }","duration":"650.5671ms","start":"2026-03-30T16:48:38.179228Z","end":"2026-03-30T16:48:38.829795Z","steps":["trace[631160304] 'agreement among raft nodes before linearized reading' (duration: 650.410898ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:38.830495Z","caller":"traceutil/trace.go:172","msg":"trace[1571113330] transaction","detail":"{read_only:false; response_revision:648; number_of_response:1; }","duration":"648.41163ms","start":"2026-03-30T16:48:38.182075Z","end":"2026-03-30T16:48:38.830487Z","steps":["trace[1571113330] 'process raft request' (duration: 648.35298ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:38.830579Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:38.182050Z","time spent":"648.474962ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:48:38.831925Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"648.057327ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:38.831959Z","caller":"traceutil/trace.go:172","msg":"trace[332839522] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:648; }","duration":"648.091818ms","start":"2026-03-30T16:48:38.183859Z","end":"2026-03-30T16:48:38.831951Z","steps":["trace[332839522] 'agreement among raft nodes before linearized reading' (duration: 648.035087ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:38.831981Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:38.183851Z","time spent":"648.125998ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:48:38.832144Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"547.328375ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T16:48:38.832169Z","caller":"traceutil/trace.go:172","msg":"trace[98497960] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:648; }","duration":"547.354236ms","start":"2026-03-30T16:48:38.284809Z","end":"2026-03-30T16:48:38.832163Z","steps":["trace[98497960] 'agreement among raft nodes before linearized reading' (duration: 547.310485ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:38.832187Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:38.284766Z","time spent":"547.416126ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":506,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:38.832203Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"229.527207ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T16:48:38.832251Z","caller":"traceutil/trace.go:172","msg":"trace[1221718147] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:648; }","duration":"229.587498ms","start":"2026-03-30T16:48:38.602652Z","end":"2026-03-30T16:48:38.832240Z","steps":["trace[1221718147] 'agreement among raft nodes before linearized reading' (duration: 229.407815ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:39.247839Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"115.218223ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T16:48:39.247918Z","caller":"traceutil/trace.go:172","msg":"trace[970847367] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:650; }","duration":"115.313804ms","start":"2026-03-30T16:48:39.132593Z","end":"2026-03-30T16:48:39.247907Z","steps":["trace[970847367] 'range keys from in-memory index tree' (duration: 113.59547ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:43.337432Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"158.543715ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:43.337597Z","caller":"traceutil/trace.go:172","msg":"trace[684884466] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:657; }","duration":"158.749049ms","start":"2026-03-30T16:48:43.178823Z","end":"2026-03-30T16:48:43.337572Z","steps":["trace[684884466] 'range keys from in-memory index tree' (duration: 158.462235ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:45.867120Z","caller":"traceutil/trace.go:172","msg":"trace[2091779239] transaction","detail":"{read_only:false; response_revision:664; number_of_response:1; }","duration":"316.841511ms","start":"2026-03-30T16:48:45.550261Z","end":"2026-03-30T16:48:45.867102Z","steps":["trace[2091779239] 'process raft request' (duration: 248.399932ms)","trace[2091779239] 'compare' (duration: 68.348187ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:45.867226Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:45.550240Z","time spent":"316.935562ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:48:45.869178Z","caller":"traceutil/trace.go:172","msg":"trace[645581592] linearizableReadLoop","detail":"{readStateIndex:714; appliedIndex:715; }","duration":"317.963406ms","start":"2026-03-30T16:48:45.551197Z","end":"2026-03-30T16:48:45.869160Z","steps":["trace[645581592] 'read index received' (duration: 317.957826ms)","trace[645581592] 'applied index is now lower than readState.Index' (duration: 4.24µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:45.869358Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"318.145088ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:45.869492Z","caller":"traceutil/trace.go:172","msg":"trace[866790083] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:664; }","duration":"318.29104ms","start":"2026-03-30T16:48:45.551191Z","end":"2026-03-30T16:48:45.869482Z","steps":["trace[866790083] 'agreement among raft nodes before linearized reading' (duration: 318.070047ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:45.869562Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:45.551175Z","time spent":"318.378852ms","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:48:45.870000Z","caller":"traceutil/trace.go:172","msg":"trace[2084103998] transaction","detail":"{read_only:false; response_revision:665; number_of_response:1; }","duration":"147.117075ms","start":"2026-03-30T16:48:45.722863Z","end":"2026-03-30T16:48:45.869980Z","steps":["trace[2084103998] 'process raft request' (duration: 146.73843ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:46.322895Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"252.363881ms","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:40"} {"level":"info","ts":"2026-03-30T16:48:46.323070Z","caller":"traceutil/trace.go:172","msg":"trace[567193546] linearizableReadLoop","detail":"{readStateIndex:717; appliedIndex:716; }","duration":"144.195509ms","start":"2026-03-30T16:48:46.178847Z","end":"2026-03-30T16:48:46.323042Z","steps":["trace[567193546] 'read index received' (duration: 1.604681ms)","trace[567193546] 'applied index is now lower than readState.Index' (duration: 142.588568ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:46.323172Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"144.308211ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:46.323199Z","caller":"traceutil/trace.go:172","msg":"trace[48877034] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:665; }","duration":"144.351071ms","start":"2026-03-30T16:48:46.178837Z","end":"2026-03-30T16:48:46.323188Z","steps":["trace[48877034] 'agreement among raft nodes before linearized reading' (duration: 144.27246ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:47.564440Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"385.656703ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:47.564538Z","caller":"traceutil/trace.go:172","msg":"trace[2005092887] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:669; }","duration":"385.880406ms","start":"2026-03-30T16:48:47.178639Z","end":"2026-03-30T16:48:47.564520Z","steps":["trace[2005092887] 'range keys from in-memory index tree' (duration: 385.604213ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:49.341332Z","caller":"traceutil/trace.go:172","msg":"trace[631911245] linearizableReadLoop","detail":"{readStateIndex:724; appliedIndex:724; }","duration":"162.92713ms","start":"2026-03-30T16:48:49.178386Z","end":"2026-03-30T16:48:49.341313Z","steps":["trace[631911245] 'read index received' (duration: 162.92285ms)","trace[631911245] 'applied index is now lower than readState.Index' (duration: 3.51µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:49.343377Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"164.971737ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:49.343455Z","caller":"traceutil/trace.go:172","msg":"trace[319143280] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:672; }","duration":"165.061838ms","start":"2026-03-30T16:48:49.178379Z","end":"2026-03-30T16:48:49.343440Z","steps":["trace[319143280] 'agreement among raft nodes before linearized reading' (duration: 163.006701ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:49.343780Z","caller":"traceutil/trace.go:172","msg":"trace[2131765386] transaction","detail":"{read_only:false; response_revision:674; number_of_response:1; }","duration":"181.930008ms","start":"2026-03-30T16:48:49.161834Z","end":"2026-03-30T16:48:49.343764Z","steps":["trace[2131765386] 'process raft request' (duration: 181.803557ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:49.343848Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"110.95505ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:48:49.343925Z","caller":"traceutil/trace.go:172","msg":"trace[1306742337] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:674; }","duration":"111.045161ms","start":"2026-03-30T16:48:49.232866Z","end":"2026-03-30T16:48:49.343911Z","steps":["trace[1306742337] 'agreement among raft nodes before linearized reading' (duration: 110.843308ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:50.749103Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.19.213.180:35464","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:51.378935Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"200.140756ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:51.378996Z","caller":"traceutil/trace.go:172","msg":"trace[829537722] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:677; }","duration":"200.218717ms","start":"2026-03-30T16:48:51.178765Z","end":"2026-03-30T16:48:51.378984Z","steps":["trace[829537722] 'range keys from in-memory index tree' (duration: 200.091416ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:51.379003Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"215.816209ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T16:48:51.379117Z","caller":"traceutil/trace.go:172","msg":"trace[1835636370] linearizableReadLoop","detail":"{readStateIndex:731; appliedIndex:730; }","duration":"228.690636ms","start":"2026-03-30T16:48:51.150409Z","end":"2026-03-30T16:48:51.379099Z","steps":["trace[1835636370] 'read index received' (duration: 14.067263ms)","trace[1835636370] 'applied index is now lower than readState.Index' (duration: 214.622163ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:51.379355Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"228.902259ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T16:48:51.379411Z","caller":"traceutil/trace.go:172","msg":"trace[699534362] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:677; }","duration":"228.99131ms","start":"2026-03-30T16:48:51.150404Z","end":"2026-03-30T16:48:51.379396Z","steps":["trace[699534362] 'agreement among raft nodes before linearized reading' (duration: 228.774907ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:48:51.413220Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"a8d4e8c653fa8dce switched to configuration voters=(4393548324105610035 12165604431944650190) learners=(14089317134902859118)"} {"level":"info","ts":"2026-03-30T16:48:51.413380Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"7b1ae51097bedd0e","local-member-id":"a8d4e8c653fa8dce","added-peer-id":"c3874f4929b5956e","added-peer-peer-urls":["https://199.19.213.180:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-30T16:48:51.413403Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:51.413429Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:51.416101Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:51.419767Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:51.420690Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:51.420729Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e","remote-peer-urls":["https://199.19.213.180:2380"]} {"level":"info","ts":"2026-03-30T16:48:51.421084Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"a8d4e8c653fa8dce","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:51.420719Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:51.420758Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"warn","ts":"2026-03-30T16:48:51.436082Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.19.213.180:35514","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:53.472505Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.19.213.180:35558","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T16:48:54.421330Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.421392Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.421474Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.435449Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"a8d4e8c653fa8dce","to":"c3874f4929b5956e","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-30T16:48:54.435582Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.438927Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"a8d4e8c653fa8dce","to":"c3874f4929b5956e","stream-type":"stream Message"} {"level":"info","ts":"2026-03-30T16:48:54.439011Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"warn","ts":"2026-03-30T16:48:54.523806Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:48:54.523874Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e","error":"EOF"} {"level":"info","ts":"2026-03-30T16:48:54.537071Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.537098Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.545081Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"a8d4e8c653fa8dce","to":"c3874f4929b5956e","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-30T16:48:54.545124Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.545136Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.545441Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"a8d4e8c653fa8dce","to":"c3874f4929b5956e","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-30T16:48:54.545474Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:54.545491Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"a8d4e8c653fa8dce","remote-peer-id":"c3874f4929b5956e"} {"level":"warn","ts":"2026-03-30T16:48:54.971528Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.19.213.180:35584","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T16:48:54.979997Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"a8d4e8c653fa8dce switched to configuration voters=(4393548324105610035 12165604431944650190 14089317134902859118)"} {"level":"info","ts":"2026-03-30T16:48:54.980329Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"7b1ae51097bedd0e","local-member-id":"a8d4e8c653fa8dce","promoted-member-id":"c3874f4929b5956e"} {"level":"info","ts":"2026-03-30T16:48:55.947165Z","caller":"traceutil/trace.go:172","msg":"trace[1868217986] linearizableReadLoop","detail":"{readStateIndex:742; appliedIndex:743; }","duration":"439.517011ms","start":"2026-03-30T16:48:55.507628Z","end":"2026-03-30T16:48:55.947145Z","steps":["trace[1868217986] 'read index received' (duration: 439.512021ms)","trace[1868217986] 'applied index is now lower than readState.Index' (duration: 3.52µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:55.947373Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"439.728275ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.227\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-30T16:48:55.947421Z","caller":"traceutil/trace.go:172","msg":"trace[420007306] range","detail":"{range_begin:/registry/masterleases/199.204.45.227; range_end:; response_count:1; response_revision:686; }","duration":"439.786695ms","start":"2026-03-30T16:48:55.507622Z","end":"2026-03-30T16:48:55.947409Z","steps":["trace[420007306] 'agreement among raft nodes before linearized reading' (duration: 439.638274ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:55.947453Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:55.507598Z","time spent":"439.846626ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.227\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:48:55.949007Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"412.574318ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:55.949053Z","caller":"traceutil/trace.go:172","msg":"trace[2134496263] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:687; }","duration":"412.643509ms","start":"2026-03-30T16:48:55.536398Z","end":"2026-03-30T16:48:55.949041Z","steps":["trace[2134496263] 'agreement among raft nodes before linearized reading' (duration: 412.547077ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:55.949112Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:48:55.536379Z","time spent":"412.726829ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:48:56.143610Z","caller":"traceutil/trace.go:172","msg":"trace[1072728006] linearizableReadLoop","detail":"{readStateIndex:745; appliedIndex:747; }","duration":"192.876766ms","start":"2026-03-30T16:48:55.950705Z","end":"2026-03-30T16:48:56.143582Z","steps":["trace[1072728006] 'read index received' (duration: 192.868486ms)","trace[1072728006] 'applied index is now lower than readState.Index' (duration: 6.81µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:48:56.143846Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"193.052508ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:48:56.143927Z","caller":"traceutil/trace.go:172","msg":"trace[2140037651] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:688; }","duration":"193.21267ms","start":"2026-03-30T16:48:55.950700Z","end":"2026-03-30T16:48:56.143912Z","steps":["trace[2140037651] 'agreement among raft nodes before linearized reading' (duration: 192.996298ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:48:57.997915Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"135.003801ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T16:49:00.483238Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"304.182154ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:00.483319Z","caller":"traceutil/trace.go:172","msg":"trace[775806573] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:717; }","duration":"304.274035ms","start":"2026-03-30T16:49:00.179031Z","end":"2026-03-30T16:49:00.483305Z","steps":["trace[775806573] 'range keys from in-memory index tree' (duration: 302.117717ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:00.483353Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"292.263094ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T16:49:00.483438Z","caller":"traceutil/trace.go:172","msg":"trace[1518043198] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:717; }","duration":"292.365775ms","start":"2026-03-30T16:49:00.191052Z","end":"2026-03-30T16:49:00.483418Z","steps":["trace[1518043198] 'range keys from in-memory index tree' (duration: 290.709025ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:00.799186Z","caller":"traceutil/trace.go:172","msg":"trace[1438428786] linearizableReadLoop","detail":"{readStateIndex:778; appliedIndex:778; }","duration":"273.242634ms","start":"2026-03-30T16:49:00.525909Z","end":"2026-03-30T16:49:00.799152Z","steps":["trace[1438428786] 'read index received' (duration: 273.235854ms)","trace[1438428786] 'applied index is now lower than readState.Index' (duration: 4.93µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:00.947833Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"421.901978ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:49:00.948006Z","caller":"traceutil/trace.go:172","msg":"trace[1414130343] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:718; }","duration":"422.08625ms","start":"2026-03-30T16:49:00.525898Z","end":"2026-03-30T16:49:00.947984Z","steps":["trace[1414130343] 'agreement among raft nodes before linearized reading' (duration: 273.352576ms)","trace[1414130343] 'range keys from in-memory index tree' (duration: 148.38559ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:00.948342Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:00.525877Z","time spent":"422.365784ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:00.948855Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"150.494417ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T16:49:01.058143Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"109.048845ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T16:49:07.244235Z","caller":"traceutil/trace.go:172","msg":"trace[2068576936] linearizableReadLoop","detail":"{readStateIndex:865; appliedIndex:866; }","duration":"120.763638ms","start":"2026-03-30T16:49:07.123456Z","end":"2026-03-30T16:49:07.244219Z","steps":["trace[2068576936] 'read index received' (duration: 120.758177ms)","trace[2068576936] 'applied index is now lower than readState.Index' (duration: 4.451µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:07.244361Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"120.90257ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:07.244394Z","caller":"traceutil/trace.go:172","msg":"trace[1997225833] range","detail":"{range_begin:/registry/resourceclaims; range_end:; response_count:0; response_revision:800; }","duration":"120.950161ms","start":"2026-03-30T16:49:07.123437Z","end":"2026-03-30T16:49:07.244387Z","steps":["trace[1997225833] 'agreement among raft nodes before linearized reading' (duration: 120.88161ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:08.284633Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"106.6097ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:08.284734Z","caller":"traceutil/trace.go:172","msg":"trace[125367567] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:802; }","duration":"106.738792ms","start":"2026-03-30T16:49:08.177975Z","end":"2026-03-30T16:49:08.284714Z","steps":["trace[125367567] 'range keys from in-memory index tree' (duration: 106.470479ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:08.285535Z","caller":"traceutil/trace.go:172","msg":"trace[2070863034] linearizableReadLoop","detail":"{readStateIndex:871; appliedIndex:869; }","duration":"281.518294ms","start":"2026-03-30T16:49:08.003990Z","end":"2026-03-30T16:49:08.285509Z","steps":["trace[2070863034] 'read index received' (duration: 277.494445ms)","trace[2070863034] 'applied index is now lower than readState.Index' (duration: 4.022579ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:08.285683Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"281.677847ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:08.285748Z","caller":"traceutil/trace.go:172","msg":"trace[1927331319] range","detail":"{range_begin:/registry/controllers; range_end:; response_count:0; response_revision:804; }","duration":"281.760988ms","start":"2026-03-30T16:49:08.003973Z","end":"2026-03-30T16:49:08.285734Z","steps":["trace[1927331319] 'agreement among raft nodes before linearized reading' (duration: 281.623626ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:12.602925Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"139.711903ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:49:14.403989Z","caller":"traceutil/trace.go:172","msg":"trace[184696649] linearizableReadLoop","detail":"{readStateIndex:893; appliedIndex:893; }","duration":"224.931052ms","start":"2026-03-30T16:49:14.179038Z","end":"2026-03-30T16:49:14.403969Z","steps":["trace[184696649] 'read index received' (duration: 224.926262ms)","trace[184696649] 'applied index is now lower than readState.Index' (duration: 3.76µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:14.404094Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"225.040983ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:14.404115Z","caller":"traceutil/trace.go:172","msg":"trace[980462350] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:823; }","duration":"225.078154ms","start":"2026-03-30T16:49:14.179031Z","end":"2026-03-30T16:49:14.404109Z","steps":["trace[980462350] 'agreement among raft nodes before linearized reading' (duration: 225.011783ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:15.749844Z","caller":"traceutil/trace.go:172","msg":"trace[1238135977] linearizableReadLoop","detail":"{readStateIndex:895; appliedIndex:896; }","duration":"433.156449ms","start":"2026-03-30T16:49:15.316666Z","end":"2026-03-30T16:49:15.749823Z","steps":["trace[1238135977] 'read index received' (duration: 433.149689ms)","trace[1238135977] 'applied index is now lower than readState.Index' (duration: 5.16µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:15.749978Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"433.309282ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:49:15.750061Z","caller":"traceutil/trace.go:172","msg":"trace[644411709] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:826; }","duration":"433.397213ms","start":"2026-03-30T16:49:15.316646Z","end":"2026-03-30T16:49:15.750043Z","steps":["trace[644411709] 'agreement among raft nodes before linearized reading' (duration: 433.269391ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:15.750115Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:15.316620Z","time spent":"433.483094ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":465,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:15.751693Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"106.149547ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"warn","ts":"2026-03-30T16:49:15.751731Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"215.946657ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:15.751782Z","caller":"traceutil/trace.go:172","msg":"trace[20544797] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:826; }","duration":"215.995168ms","start":"2026-03-30T16:49:15.535762Z","end":"2026-03-30T16:49:15.751757Z","steps":["trace[20544797] 'agreement among raft nodes before linearized reading' (duration: 215.911627ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:15.751746Z","caller":"traceutil/trace.go:172","msg":"trace[557196081] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:826; }","duration":"106.206908ms","start":"2026-03-30T16:49:15.645528Z","end":"2026-03-30T16:49:15.751735Z","steps":["trace[557196081] 'agreement among raft nodes before linearized reading' (duration: 106.044216ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:15.751877Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"243.447926ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.227\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-30T16:49:15.751925Z","caller":"traceutil/trace.go:172","msg":"trace[817197933] range","detail":"{range_begin:/registry/masterleases/199.204.45.227; range_end:; response_count:1; response_revision:826; }","duration":"243.509947ms","start":"2026-03-30T16:49:15.508407Z","end":"2026-03-30T16:49:15.751917Z","steps":["trace[817197933] 'agreement among raft nodes before linearized reading' (duration: 243.352245ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:15.752220Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"326.307174ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T16:49:15.752318Z","caller":"traceutil/trace.go:172","msg":"trace[429941567] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:826; }","duration":"326.417325ms","start":"2026-03-30T16:49:15.425880Z","end":"2026-03-30T16:49:15.752298Z","steps":["trace[429941567] 'agreement among raft nodes before linearized reading' (duration: 325.620996ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:15.752398Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:15.425854Z","time spent":"326.523926ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":506,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:15.905911Z","caller":"traceutil/trace.go:172","msg":"trace[1193921598] linearizableReadLoop","detail":"{readStateIndex:896; appliedIndex:896; }","duration":"151.72716ms","start":"2026-03-30T16:49:15.754137Z","end":"2026-03-30T16:49:15.905864Z","steps":["trace[1193921598] 'read index received' (duration: 151.71844ms)","trace[1193921598] 'applied index is now lower than readState.Index' (duration: 7.23µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:16.052123Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"298.021354ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:16.052185Z","caller":"traceutil/trace.go:172","msg":"trace[1663702121] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:826; }","duration":"298.092955ms","start":"2026-03-30T16:49:15.754080Z","end":"2026-03-30T16:49:16.052173Z","steps":["trace[1663702121] 'agreement among raft nodes before linearized reading' (duration: 151.917442ms)","trace[1663702121] 'range keys from in-memory index tree' (duration: 146.078511ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:16.052302Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"146.370935ms","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:40"} {"level":"info","ts":"2026-03-30T16:49:16.052413Z","caller":"traceutil/trace.go:172","msg":"trace[118720973] linearizableReadLoop","detail":"{readStateIndex:897; appliedIndex:896; }","duration":"146.398796ms","start":"2026-03-30T16:49:15.905996Z","end":"2026-03-30T16:49:16.052395Z","steps":["trace[118720973] 'read index received' (duration: 1.197955ms)","trace[118720973] 'applied index is now lower than readState.Index' (duration: 145.199511ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:16.052489Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"249.850939ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattributesclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:16.052514Z","caller":"traceutil/trace.go:172","msg":"trace[2014577072] range","detail":"{range_begin:/registry/volumeattributesclasses; range_end:; response_count:0; response_revision:826; }","duration":"249.882031ms","start":"2026-03-30T16:49:15.802622Z","end":"2026-03-30T16:49:16.052504Z","steps":["trace[2014577072] 'agreement among raft nodes before linearized reading' (duration: 249.819299ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:16.315956Z","caller":"traceutil/trace.go:172","msg":"trace[338170223] linearizableReadLoop","detail":"{readStateIndex:898; appliedIndex:898; }","duration":"137.506287ms","start":"2026-03-30T16:49:16.178425Z","end":"2026-03-30T16:49:16.315932Z","steps":["trace[338170223] 'read index received' (duration: 137.471597ms)","trace[338170223] 'applied index is now lower than readState.Index' (duration: 5.19µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:16.442381Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"263.969782ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:49:16.442577Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"127.476648ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:49:16.442666Z","caller":"traceutil/trace.go:172","msg":"trace[1598858859] transaction","detail":"{read_only:false; response_revision:827; number_of_response:1; }","duration":"389.216581ms","start":"2026-03-30T16:49:16.053434Z","end":"2026-03-30T16:49:16.442650Z","steps":["trace[1598858859] 'process raft request' (duration: 261.609062ms)","trace[1598858859] 'compare' (duration: 127.387507ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:49:16.442692Z","caller":"traceutil/trace.go:172","msg":"trace[827238877] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:826; }","duration":"264.048953ms","start":"2026-03-30T16:49:16.178386Z","end":"2026-03-30T16:49:16.442435Z","steps":["trace[827238877] 'agreement among raft nodes before linearized reading' (duration: 137.637328ms)","trace[827238877] 'range keys from in-memory index tree' (duration: 126.311764ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:16.442709Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:16.053413Z","time spent":"389.276573ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:49:16.944531Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228003942,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:17.080448Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"131.911718ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:49:17.256089Z","caller":"traceutil/trace.go:172","msg":"trace[281240730] linearizableReadLoop","detail":"{readStateIndex:901; appliedIndex:901; }","duration":"811.640049ms","start":"2026-03-30T16:49:16.444416Z","end":"2026-03-30T16:49:17.256056Z","steps":["trace[281240730] 'read index received' (duration: 811.632499ms)","trace[281240730] 'applied index is now lower than readState.Index' (duration: 6.17µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:17.257712Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"813.28502ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:460"} {"level":"info","ts":"2026-03-30T16:49:17.257775Z","caller":"traceutil/trace.go:172","msg":"trace[1282592279] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:829; }","duration":"813.354641ms","start":"2026-03-30T16:49:16.444405Z","end":"2026-03-30T16:49:17.257759Z","steps":["trace[1282592279] 'agreement among raft nodes before linearized reading' (duration: 811.760032ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:17.257806Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:16.444388Z","time spent":"813.410221ms","remote":"127.0.0.1:55346","response type":"/etcdserverpb.KV/Range","request count":0,"request size":51,"response count":1,"response size":483,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:17.258874Z","caller":"traceutil/trace.go:172","msg":"trace[1069545463] transaction","detail":"{read_only:false; response_revision:831; number_of_response:1; }","duration":"701.690891ms","start":"2026-03-30T16:49:16.557173Z","end":"2026-03-30T16:49:17.258864Z","steps":["trace[1069545463] 'process raft request' (duration: 700.526397ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:17.258951Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:16.557144Z","time spent":"701.761241ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:49:17.259611Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"748.139214ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:49:17.259639Z","caller":"traceutil/trace.go:172","msg":"trace[209158787] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:832; }","duration":"748.168695ms","start":"2026-03-30T16:49:16.511464Z","end":"2026-03-30T16:49:17.259633Z","steps":["trace[209158787] 'agreement among raft nodes before linearized reading' (duration: 748.094584ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:17.259658Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:16.511443Z","time spent":"748.210445ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:17.259720Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"798.108288ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaimtemplates\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:17.259744Z","caller":"traceutil/trace.go:172","msg":"trace[285813917] range","detail":"{range_begin:/registry/resourceclaimtemplates; range_end:; response_count:0; response_revision:832; }","duration":"798.133108ms","start":"2026-03-30T16:49:16.461604Z","end":"2026-03-30T16:49:17.259737Z","steps":["trace[285813917] 'agreement among raft nodes before linearized reading' (duration: 798.093017ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:17.259749Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"361.614275ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/roles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:49:17.259762Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:16.461584Z","time spent":"798.173468ms","remote":"127.0.0.1:56026","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":28,"request content":"key:\"/registry/resourceclaimtemplates\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:17.259768Z","caller":"traceutil/trace.go:172","msg":"trace[2086569020] range","detail":"{range_begin:/registry/roles; range_end:; response_count:0; response_revision:832; }","duration":"361.634115ms","start":"2026-03-30T16:49:16.898129Z","end":"2026-03-30T16:49:17.259763Z","steps":["trace[2086569020] 'agreement among raft nodes before linearized reading' (duration: 361.600945ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:17.259784Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:16.898099Z","time spent":"361.680925ms","remote":"127.0.0.1:55636","response type":"/etcdserverpb.KV/Range","request count":0,"request size":19,"response count":0,"response size":28,"request content":"key:\"/registry/roles\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:17.259884Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"707.896076ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterrolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:17.259909Z","caller":"traceutil/trace.go:172","msg":"trace[238075816] range","detail":"{range_begin:/registry/clusterrolebindings; range_end:; response_count:0; response_revision:832; }","duration":"707.920656ms","start":"2026-03-30T16:49:16.551981Z","end":"2026-03-30T16:49:17.259902Z","steps":["trace[238075816] 'agreement among raft nodes before linearized reading' (duration: 707.884265ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:17.259925Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:16.551964Z","time spent":"707.955836ms","remote":"127.0.0.1:55676","response type":"/etcdserverpb.KV/Range","request count":0,"request size":33,"response count":0,"response size":28,"request content":"key:\"/registry/clusterrolebindings\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:40.706955Z","caller":"traceutil/trace.go:172","msg":"trace[1221404512] linearizableReadLoop","detail":"{readStateIndex:1057; appliedIndex:1057; }","duration":"116.246922ms","start":"2026-03-30T16:49:40.590666Z","end":"2026-03-30T16:49:40.706913Z","steps":["trace[1221404512] 'read index received' (duration: 116.241372ms)","trace[1221404512] 'applied index is now lower than readState.Index' (duration: 4.35µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:40.707214Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"116.529906ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:14991"} {"level":"info","ts":"2026-03-30T16:49:40.707292Z","caller":"traceutil/trace.go:172","msg":"trace[398511948] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:970; }","duration":"116.616177ms","start":"2026-03-30T16:49:40.590657Z","end":"2026-03-30T16:49:40.707274Z","steps":["trace[398511948] 'agreement among raft nodes before linearized reading' (duration: 116.391483ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:42.653885Z","caller":"traceutil/trace.go:172","msg":"trace[630454727] linearizableReadLoop","detail":"{readStateIndex:1074; appliedIndex:1074; }","duration":"474.356151ms","start":"2026-03-30T16:49:42.179511Z","end":"2026-03-30T16:49:42.653867Z","steps":["trace[630454727] 'read index received' (duration: 474.350971ms)","trace[630454727] 'applied index is now lower than readState.Index' (duration: 3.91µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:42.654025Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"474.500143ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:42.654051Z","caller":"traceutil/trace.go:172","msg":"trace[2056288970] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:984; }","duration":"474.538794ms","start":"2026-03-30T16:49:42.179505Z","end":"2026-03-30T16:49:42.654044Z","steps":["trace[2056288970] 'agreement among raft nodes before linearized reading' (duration: 474.482013ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:42.655664Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"201.892832ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:49:42.655709Z","caller":"traceutil/trace.go:172","msg":"trace[616303020] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:984; }","duration":"201.941382ms","start":"2026-03-30T16:49:42.453758Z","end":"2026-03-30T16:49:42.655699Z","steps":["trace[616303020] 'agreement among raft nodes before linearized reading' (duration: 201.801901ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:43.826609Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004099,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:44.329811Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004099,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:44.830233Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004099,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:45.091589Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.912529943s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:45.091654Z","caller":"traceutil/trace.go:172","msg":"trace[1297339516] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:984; }","duration":"1.912608994s","start":"2026-03-30T16:49:43.179033Z","end":"2026-03-30T16:49:45.091642Z","steps":["trace[1297339516] 'range keys from in-memory index tree' (duration: 1.912474303s)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:45.110783Z","caller":"traceutil/trace.go:172","msg":"trace[1981460137] linearizableReadLoop","detail":"{readStateIndex:1089; appliedIndex:1076; }","duration":"1.784412266s","start":"2026-03-30T16:49:43.326330Z","end":"2026-03-30T16:49:45.110742Z","steps":["trace[1981460137] 'read index received' (duration: 1.782069011s)","trace[1981460137] 'applied index is now lower than readState.Index' (duration: 2.342035ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:45.111272Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.784935662s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T16:49:45.111323Z","caller":"traceutil/trace.go:172","msg":"trace[1328625607] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:998; }","duration":"1.784982363s","start":"2026-03-30T16:49:43.326314Z","end":"2026-03-30T16:49:45.111296Z","steps":["trace[1328625607] 'agreement among raft nodes before linearized reading' (duration: 1.784830051s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.111356Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:43.326287Z","time spent":"1.785060384s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:45.113710Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"630.978638ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csinodes\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:45.113794Z","caller":"traceutil/trace.go:172","msg":"trace[781922656] range","detail":"{range_begin:/registry/csinodes; range_end:; response_count:0; response_revision:998; }","duration":"631.059789ms","start":"2026-03-30T16:49:44.482716Z","end":"2026-03-30T16:49:45.113776Z","steps":["trace[781922656] 'agreement among raft nodes before linearized reading' (duration: 630.951097ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.113847Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:44.482697Z","time spent":"631.125969ms","remote":"127.0.0.1:55754","response type":"/etcdserverpb.KV/Range","request count":0,"request size":22,"response count":0,"response size":28,"request content":"key:\"/registry/csinodes\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:45.114269Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.394551637s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/priorityclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:45.114387Z","caller":"traceutil/trace.go:172","msg":"trace[1452434798] range","detail":"{range_begin:/registry/priorityclasses; range_end:; response_count:0; response_revision:998; }","duration":"1.394652028s","start":"2026-03-30T16:49:43.719701Z","end":"2026-03-30T16:49:45.114353Z","steps":["trace[1452434798] 'agreement among raft nodes before linearized reading' (duration: 1.394510116s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.114429Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:43.719679Z","time spent":"1.394739268s","remote":"127.0.0.1:55698","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":28,"request content":"key:\"/registry/priorityclasses\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:45.114264Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"802.073588ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:49:45.114777Z","caller":"traceutil/trace.go:172","msg":"trace[661148054] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:998; }","duration":"802.624214ms","start":"2026-03-30T16:49:44.312128Z","end":"2026-03-30T16:49:45.114752Z","steps":["trace[661148054] 'agreement among raft nodes before linearized reading' (duration: 801.911326ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.115021Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:44.312106Z","time spent":"802.891647ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":465,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:45.113297Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.580171202s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:45.116105Z","caller":"traceutil/trace.go:172","msg":"trace[853017505] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:998; }","duration":"1.582977273s","start":"2026-03-30T16:49:43.533110Z","end":"2026-03-30T16:49:45.116088Z","steps":["trace[853017505] 'agreement among raft nodes before linearized reading' (duration: 1.580105142s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.116275Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:43.533091Z","time spent":"1.583094244s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:49:45.448022Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"130.666969ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:49:45.448935Z","caller":"traceutil/trace.go:172","msg":"trace[193501275] linearizableReadLoop","detail":"{readStateIndex:1089; appliedIndex:1090; }","duration":"332.220572ms","start":"2026-03-30T16:49:45.116685Z","end":"2026-03-30T16:49:45.448905Z","steps":["trace[193501275] 'read index received' (duration: 332.207212ms)","trace[193501275] 'applied index is now lower than readState.Index' (duration: 10.32µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:45.449131Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"332.423734ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:45.449180Z","caller":"traceutil/trace.go:172","msg":"trace[610661274] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:999; }","duration":"332.488965ms","start":"2026-03-30T16:49:45.116676Z","end":"2026-03-30T16:49:45.449165Z","steps":["trace[610661274] 'agreement among raft nodes before linearized reading' (duration: 332.379474ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:45.450264Z","caller":"traceutil/trace.go:172","msg":"trace[561962828] transaction","detail":"{read_only:false; number_of_response:0; response_revision:1005; }","duration":"330.920999ms","start":"2026-03-30T16:49:45.119334Z","end":"2026-03-30T16:49:45.450255Z","steps":["trace[561962828] 'process raft request' (duration: 330.907369ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.450339Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.119326Z","time spent":"330.976469ms","remote":"127.0.0.1:56066","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":28,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T16:49:45.450354Z","caller":"traceutil/trace.go:172","msg":"trace[389511981] transaction","detail":"{read_only:false; number_of_response:0; response_revision:1005; }","duration":"331.144692ms","start":"2026-03-30T16:49:45.119191Z","end":"2026-03-30T16:49:45.450335Z","steps":["trace[389511981] 'process raft request' (duration: 331.03885ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.450628Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.119175Z","time spent":"331.283244ms","remote":"127.0.0.1:56066","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":28,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:49:45.945444Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"129.712948ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:7343"} {"level":"info","ts":"2026-03-30T16:49:45.945753Z","caller":"traceutil/trace.go:172","msg":"trace[1771103574] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1005; }","duration":"826.130419ms","start":"2026-03-30T16:49:45.119597Z","end":"2026-03-30T16:49:45.945728Z","steps":["trace[1771103574] 'process raft request' (duration: 695.892315ms)","trace[1771103574] 'compare' (duration: 129.651537ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:45.945895Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.119575Z","time spent":"826.269921ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":102,"response count":0,"response size":7366,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:45.945897Z","caller":"traceutil/trace.go:172","msg":"trace[1974792256] transaction","detail":"{read_only:false; number_of_response:0; response_revision:1005; }","duration":"825.043288ms","start":"2026-03-30T16:49:45.120831Z","end":"2026-03-30T16:49:45.945875Z","steps":["trace[1974792256] 'process raft request' (duration: 824.827616ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.945989Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.120822Z","time spent":"825.133039ms","remote":"127.0.0.1:56066","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":28,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T16:49:45.946018Z","caller":"traceutil/trace.go:172","msg":"trace[2057226434] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1005; }","duration":"824.481882ms","start":"2026-03-30T16:49:45.121502Z","end":"2026-03-30T16:49:45.945984Z","steps":["trace[2057226434] 'process raft request' (duration: 824.31232ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.946197Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.121477Z","time spent":"824.623933ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":102,"response count":0,"response size":7366,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:45.946298Z","caller":"traceutil/trace.go:172","msg":"trace[1371730512] transaction","detail":"{read_only:false; response_revision:1006; number_of_response:1; }","duration":"823.737292ms","start":"2026-03-30T16:49:45.122551Z","end":"2026-03-30T16:49:45.946288Z","steps":["trace[1371730512] 'process raft request' (duration: 823.346868ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.946342Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.122542Z","time spent":"823.779863ms","remote":"127.0.0.1:56066","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":998,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T16:49:45.946608Z","caller":"traceutil/trace.go:172","msg":"trace[1429117023] transaction","detail":"{read_only:false; response_revision:1007; number_of_response:1; }","duration":"823.045915ms","start":"2026-03-30T16:49:45.123532Z","end":"2026-03-30T16:49:45.946578Z","steps":["trace[1429117023] 'process raft request' (duration: 822.55299ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.946740Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.123512Z","time spent":"823.168437ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":10063,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:45.947087Z","caller":"traceutil/trace.go:172","msg":"trace[1162706541] transaction","detail":"{read_only:false; response_revision:1008; number_of_response:1; }","duration":"819.659178ms","start":"2026-03-30T16:49:45.127414Z","end":"2026-03-30T16:49:45.947073Z","steps":["trace[1162706541] 'process raft request' (duration: 819.574637ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:45.947563Z","caller":"traceutil/trace.go:172","msg":"trace[2131929851] transaction","detail":"{read_only:false; response_revision:1009; number_of_response:1; }","duration":"818.93903ms","start":"2026-03-30T16:49:45.128590Z","end":"2026-03-30T16:49:45.947529Z","steps":["trace[2131929851] 'process raft request' (duration: 818.543615ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.947171Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.127397Z","time spent":"819.735598ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":30374,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:49:45.948320Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.128579Z","time spent":"819.312714ms","remote":"127.0.0.1:56066","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":998,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:49:45.949502Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004113,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:49:45.950128Z","caller":"traceutil/trace.go:172","msg":"trace[400462363] transaction","detail":"{read_only:false; response_revision:1010; number_of_response:1; }","duration":"819.536985ms","start":"2026-03-30T16:49:45.130574Z","end":"2026-03-30T16:49:45.950111Z","steps":["trace[400462363] 'process raft request' (duration: 819.110671ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.950196Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.129996Z","time spent":"820.166263ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":30393,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:45.950309Z","caller":"traceutil/trace.go:172","msg":"trace[1344321366] transaction","detail":"{read_only:false; response_revision:1011; number_of_response:1; }","duration":"815.348429ms","start":"2026-03-30T16:49:45.134941Z","end":"2026-03-30T16:49:45.950289Z","steps":["trace[1344321366] 'process raft request' (duration: 815.094617ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.950419Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.134913Z","time spent":"815.446631ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":28739,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:45.979334Z","caller":"traceutil/trace.go:172","msg":"trace[1248368228] linearizableReadLoop","detail":"{readStateIndex:1122; appliedIndex:1125; }","duration":"530.299141ms","start":"2026-03-30T16:49:45.449017Z","end":"2026-03-30T16:49:45.979316Z","steps":["trace[1248368228] 'read index received' (duration: 530.291321ms)","trace[1248368228] 'applied index is now lower than readState.Index' (duration: 6.54µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:45.979503Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"861.997748ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:45.979527Z","caller":"traceutil/trace.go:172","msg":"trace[706524223] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1013; }","duration":"862.036868ms","start":"2026-03-30T16:49:45.117483Z","end":"2026-03-30T16:49:45.979520Z","steps":["trace[706524223] 'agreement among raft nodes before linearized reading' (duration: 861.959378ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.979550Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.117464Z","time spent":"862.081239ms","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:49:45.980078Z","caller":"traceutil/trace.go:172","msg":"trace[893489531] transaction","detail":"{read_only:false; response_revision:1014; number_of_response:1; }","duration":"523.006759ms","start":"2026-03-30T16:49:45.457053Z","end":"2026-03-30T16:49:45.980060Z","steps":["trace[893489531] 'process raft request' (duration: 522.911048ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:45.980340Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.457036Z","time spent":"523.09971ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7326,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:49:46.059152Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004120,"received-request-id":10218277501228004113} {"level":"info","ts":"2026-03-30T16:49:46.059803Z","caller":"traceutil/trace.go:172","msg":"trace[1986557557] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1020; }","duration":"101.471354ms","start":"2026-03-30T16:49:45.958308Z","end":"2026-03-30T16:49:46.059779Z","steps":["trace[1986557557] 'process raft request' (duration: 101.090979ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:46.085447Z","caller":"traceutil/trace.go:172","msg":"trace[1174648436] linearizableReadLoop","detail":"{readStateIndex:1134; appliedIndex:1138; }","duration":"106.021122ms","start":"2026-03-30T16:49:45.979408Z","end":"2026-03-30T16:49:46.085430Z","steps":["trace[1174648436] 'read index received' (duration: 106.013372ms)","trace[1174648436] 'applied index is now lower than readState.Index' (duration: 6.83µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:46.085611Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"137.698894ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1beta1.gateway.networking.k8s.io\" limit:1 ","response":"range_response_count:1 size:1010"} {"level":"info","ts":"2026-03-30T16:49:46.085633Z","caller":"traceutil/trace.go:172","msg":"trace[743425089] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1beta1.gateway.networking.k8s.io; range_end:; response_count:1; response_revision:1021; }","duration":"137.735984ms","start":"2026-03-30T16:49:45.947891Z","end":"2026-03-30T16:49:46.085627Z","steps":["trace[743425089] 'agreement among raft nodes before linearized reading' (duration: 137.620083ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.085639Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"632.058975ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1.gateway.networking.k8s.io\" limit:1 ","response":"range_response_count:1 size:995"} {"level":"info","ts":"2026-03-30T16:49:46.085696Z","caller":"traceutil/trace.go:172","msg":"trace[1551722539] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1.gateway.networking.k8s.io; range_end:; response_count:1; response_revision:1021; }","duration":"632.129307ms","start":"2026-03-30T16:49:45.453554Z","end":"2026-03-30T16:49:46.085684Z","steps":["trace[1551722539] 'agreement among raft nodes before linearized reading' (duration: 631.971584ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.085726Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"633.799326ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:49:46.085803Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.453526Z","time spent":"632.222058ms","remote":"127.0.0.1:56066","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":1018,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices/v1.gateway.networking.k8s.io\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:46.085826Z","caller":"traceutil/trace.go:172","msg":"trace[546777800] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1021; }","duration":"633.910967ms","start":"2026-03-30T16:49:45.451900Z","end":"2026-03-30T16:49:46.085811Z","steps":["trace[546777800] 'agreement among raft nodes before linearized reading' (duration: 633.756095ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.085950Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"576.133546ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.227\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-30T16:49:46.085974Z","caller":"traceutil/trace.go:172","msg":"trace[485407216] range","detail":"{range_begin:/registry/masterleases/199.204.45.227; range_end:; response_count:1; response_revision:1021; }","duration":"576.156976ms","start":"2026-03-30T16:49:45.509811Z","end":"2026-03-30T16:49:46.085968Z","steps":["trace[485407216] 'agreement among raft nodes before linearized reading' (duration: 576.093475ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.086028Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"267.412733ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T16:49:46.086244Z","caller":"traceutil/trace.go:172","msg":"trace[2064703511] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1021; }","duration":"267.672286ms","start":"2026-03-30T16:49:45.818555Z","end":"2026-03-30T16:49:46.086227Z","steps":["trace[2064703511] 'agreement among raft nodes before linearized reading' (duration: 267.185061ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.086203Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"634.789657ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1alpha1.gateway.networking.x-k8s.io\" limit:1 ","response":"range_response_count:1 size:1019"} {"level":"info","ts":"2026-03-30T16:49:46.086308Z","caller":"traceutil/trace.go:172","msg":"trace[1988771881] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1alpha1.gateway.networking.x-k8s.io; range_end:; response_count:1; response_revision:1021; }","duration":"634.890208ms","start":"2026-03-30T16:49:45.451407Z","end":"2026-03-30T16:49:46.086297Z","steps":["trace[1988771881] 'agreement among raft nodes before linearized reading' (duration: 634.754576ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.086302Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.509790Z","time spent":"576.196696ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.227\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:46.086330Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:45.451393Z","time spent":"634.931108ms","remote":"127.0.0.1:56066","response type":"/etcdserverpb.KV/Range","request count":0,"request size":85,"response count":1,"response size":1042,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices/v1alpha1.gateway.networking.x-k8s.io\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:46.086656Z","caller":"traceutil/trace.go:172","msg":"trace[1492551352] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1021; }","duration":"112.472054ms","start":"2026-03-30T16:49:45.974176Z","end":"2026-03-30T16:49:46.086648Z","steps":["trace[1492551352] 'process raft request' (duration: 112.277582ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:46.088053Z","caller":"traceutil/trace.go:172","msg":"trace[2076168903] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1021; }","duration":"107.127505ms","start":"2026-03-30T16:49:45.980916Z","end":"2026-03-30T16:49:46.088044Z","steps":["trace[2076168903] 'process raft request' (duration: 107.023194ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:46.195871Z","caller":"traceutil/trace.go:172","msg":"trace[1337402810] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1024; }","duration":"129.128879ms","start":"2026-03-30T16:49:46.066711Z","end":"2026-03-30T16:49:46.195840Z","steps":["trace[1337402810] 'process raft request' (duration: 128.960817ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:46.196085Z","caller":"traceutil/trace.go:172","msg":"trace[1004379512] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1024; }","duration":"117.594601ms","start":"2026-03-30T16:49:46.078479Z","end":"2026-03-30T16:49:46.196074Z","steps":["trace[1004379512] 'process raft request' (duration: 117.4257ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:46.196496Z","caller":"traceutil/trace.go:172","msg":"trace[389805249] linearizableReadLoop","detail":"{readStateIndex:1152; appliedIndex:1162; }","duration":"110.997509ms","start":"2026-03-30T16:49:46.085492Z","end":"2026-03-30T16:49:46.196490Z","steps":["trace[389805249] 'read index received' (duration: 110.994099ms)","trace[389805249] 'applied index is now lower than readState.Index' (duration: 2.71µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:46.196583Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"209.026354ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:46.196603Z","caller":"traceutil/trace.go:172","msg":"trace[1165065171] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1028; }","duration":"209.059494ms","start":"2026-03-30T16:49:45.987538Z","end":"2026-03-30T16:49:46.196598Z","steps":["trace[1165065171] 'agreement among raft nodes before linearized reading' (duration: 208.991363ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:46.196030Z","caller":"traceutil/trace.go:172","msg":"trace[548252945] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1024; }","duration":"124.337207ms","start":"2026-03-30T16:49:46.071652Z","end":"2026-03-30T16:49:46.195990Z","steps":["trace[548252945] 'process raft request' (duration: 124.208975ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.727652Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004133,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:49:46.966196Z","caller":"traceutil/trace.go:172","msg":"trace[479088931] linearizableReadLoop","detail":"{readStateIndex:1174; appliedIndex:1178; }","duration":"739.062848ms","start":"2026-03-30T16:49:46.227114Z","end":"2026-03-30T16:49:46.966176Z","steps":["trace[479088931] 'read index received' (duration: 739.058368ms)","trace[479088931] 'applied index is now lower than readState.Index' (duration: 3.73µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:46.966339Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"768.006998ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:46.966388Z","caller":"traceutil/trace.go:172","msg":"trace[1084820104] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1035; }","duration":"768.066879ms","start":"2026-03-30T16:49:46.198312Z","end":"2026-03-30T16:49:46.966379Z","steps":["trace[1084820104] 'agreement among raft nodes before linearized reading' (duration: 767.979578ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.966419Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:46.198303Z","time spent":"768.109879ms","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:49:46.966427Z","caller":"traceutil/trace.go:172","msg":"trace[1225341211] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1035; }","duration":"726.102714ms","start":"2026-03-30T16:49:46.240309Z","end":"2026-03-30T16:49:46.966412Z","steps":["trace[1225341211] 'process raft request' (duration: 725.958233ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.966608Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:46.240304Z","time spent":"726.187295ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":96,"response count":0,"response size":31172,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:46.966783Z","caller":"traceutil/trace.go:172","msg":"trace[1848154155] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1035; }","duration":"726.549509ms","start":"2026-03-30T16:49:46.240207Z","end":"2026-03-30T16:49:46.966757Z","steps":["trace[1848154155] 'process raft request' (duration: 726.013373ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:46.967152Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:46.240192Z","time spent":"726.773521ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":112,"response count":0,"response size":23117,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:49:47.006695Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004136,"received-request-id":10218277501228004133} {"level":"warn","ts":"2026-03-30T16:49:47.038663Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"504.285529ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:49:47.038723Z","caller":"traceutil/trace.go:172","msg":"trace[1286140808] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1040; }","duration":"504.3582ms","start":"2026-03-30T16:49:46.534352Z","end":"2026-03-30T16:49:47.038710Z","steps":["trace[1286140808] 'agreement among raft nodes before linearized reading' (duration: 504.182748ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:47.038703Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"808.427296ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:460"} {"level":"warn","ts":"2026-03-30T16:49:47.038751Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:46.534325Z","time spent":"504.420061ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:47.038786Z","caller":"traceutil/trace.go:172","msg":"trace[774893523] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1040; }","duration":"808.512787ms","start":"2026-03-30T16:49:46.230252Z","end":"2026-03-30T16:49:47.038765Z","steps":["trace[774893523] 'agreement among raft nodes before linearized reading' (duration: 808.289154ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:47.038698Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"412.636026ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:49:47.038838Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:46.230231Z","time spent":"808.594517ms","remote":"127.0.0.1:55346","response type":"/etcdserverpb.KV/Range","request count":0,"request size":51,"response count":1,"response size":483,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:47.038868Z","caller":"traceutil/trace.go:172","msg":"trace[1145654135] range","detail":"{range_begin:/registry/validatingadmissionpolicybindings; range_end:; response_count:0; response_revision:1040; }","duration":"412.889298ms","start":"2026-03-30T16:49:46.625957Z","end":"2026-03-30T16:49:47.038847Z","steps":["trace[1145654135] 'agreement among raft nodes before linearized reading' (duration: 412.597625ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:47.038929Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:46.625935Z","time spent":"412.973519ms","remote":"127.0.0.1:55964","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":28,"request content":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:48.148244Z","caller":"traceutil/trace.go:172","msg":"trace[1140026759] transaction","detail":"{read_only:false; response_revision:1066; number_of_response:1; }","duration":"445.927001ms","start":"2026-03-30T16:49:47.702303Z","end":"2026-03-30T16:49:48.148230Z","steps":["trace[1140026759] 'process raft request' (duration: 445.733409ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:48.148383Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:47.702275Z","time spent":"446.031652ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:48.148658Z","caller":"traceutil/trace.go:172","msg":"trace[302862497] transaction","detail":"{read_only:false; response_revision:1065; number_of_response:1; }","duration":"502.537847ms","start":"2026-03-30T16:49:47.646091Z","end":"2026-03-30T16:49:48.148629Z","steps":["trace[302862497] 'process raft request' (duration: 501.812339ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:48.148739Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:47.646063Z","time spent":"502.640438ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":65421,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:48.148812Z","caller":"traceutil/trace.go:172","msg":"trace[1951062982] transaction","detail":"{read_only:false; response_revision:1064; number_of_response:1; }","duration":"541.575858ms","start":"2026-03-30T16:49:47.607227Z","end":"2026-03-30T16:49:48.148803Z","steps":["trace[1951062982] 'process raft request' (duration: 540.574357ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:48.148857Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:47.607208Z","time spent":"541.623338ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":20842,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:49:49.075065Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004165,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:49:49.213518Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"a8d4e8c653fa8dce is starting a new election at term 2"} {"level":"info","ts":"2026-03-30T16:49:49.213593Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"a8d4e8c653fa8dce became pre-candidate at term 2"} {"level":"info","ts":"2026-03-30T16:49:49.213620Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"a8d4e8c653fa8dce [logterm: 2, index: 1250] sent MsgPreVote request to 3cf904ec39d3e333 at term 2"} {"level":"info","ts":"2026-03-30T16:49:49.213638Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"a8d4e8c653fa8dce [logterm: 2, index: 1250] sent MsgPreVote request to c3874f4929b5956e at term 2"} {"level":"info","ts":"2026-03-30T16:49:49.213653Z","logger":"raft","caller":"v3@v3.6.0/node.go:376","msg":"raft.node: a8d4e8c653fa8dce lost leader 3cf904ec39d3e333 at term 2"} {"level":"info","ts":"2026-03-30T16:49:49.213698Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"a8d4e8c653fa8dce received MsgPreVoteResp from a8d4e8c653fa8dce at term 2"} {"level":"info","ts":"2026-03-30T16:49:49.213715Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"a8d4e8c653fa8dce has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T16:49:49.215063Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"a8d4e8c653fa8dce received MsgPreVoteResp from c3874f4929b5956e at term 2"} {"level":"info","ts":"2026-03-30T16:49:49.215213Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"a8d4e8c653fa8dce has received 2 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T16:49:49.215302Z","logger":"raft","caller":"v3@v3.6.0/raft.go:912","msg":"a8d4e8c653fa8dce became candidate at term 3"} {"level":"info","ts":"2026-03-30T16:49:49.215398Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"a8d4e8c653fa8dce [logterm: 2, index: 1250] sent MsgVote request to 3cf904ec39d3e333 at term 3"} {"level":"info","ts":"2026-03-30T16:49:49.215483Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"a8d4e8c653fa8dce [logterm: 2, index: 1250] sent MsgVote request to c3874f4929b5956e at term 3"} {"level":"info","ts":"2026-03-30T16:49:49.449522Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"a8d4e8c653fa8dce received MsgVoteResp from a8d4e8c653fa8dce at term 3"} {"level":"info","ts":"2026-03-30T16:49:49.449959Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"a8d4e8c653fa8dce has received 1 MsgVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-30T16:49:49.575848Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004165,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:49:49.624106Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"a8d4e8c653fa8dce received MsgVoteResp from c3874f4929b5956e at term 3"} {"level":"info","ts":"2026-03-30T16:49:49.624185Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"a8d4e8c653fa8dce has received 2 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T16:49:49.624230Z","logger":"raft","caller":"v3@v3.6.0/raft.go:970","msg":"a8d4e8c653fa8dce became leader at term 3"} {"level":"info","ts":"2026-03-30T16:49:49.624253Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: a8d4e8c653fa8dce elected leader a8d4e8c653fa8dce at term 3"} {"level":"warn","ts":"2026-03-30T16:49:49.624425Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.050216083s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllers\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-30T16:49:49.624531Z","caller":"traceutil/trace.go:172","msg":"trace[378400700] range","detail":"{range_begin:/registry/controllers; range_end:; }","duration":"1.050335114s","start":"2026-03-30T16:49:48.574176Z","end":"2026-03-30T16:49:49.624511Z","steps":["trace[378400700] 'agreement among raft nodes before linearized reading' (duration: 1.050214733s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:49.624612Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:48.574156Z","time spent":"1.050433755s","remote":"127.0.0.1:55440","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":0,"request content":"key:\"/registry/controllers\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:49.701274Z","caller":"etcdserver/v3_server.go:903","msg":"first commit in current term: resending ReadIndex request"} {"level":"warn","ts":"2026-03-30T16:49:49.702284Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"165.868726ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:49:49.702408Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"523.352708ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:49:49.702415Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"437.613455ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:49:49.702530Z","caller":"traceutil/trace.go:172","msg":"trace[200863265] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1074; }","duration":"437.772806ms","start":"2026-03-30T16:49:49.264736Z","end":"2026-03-30T16:49:49.702509Z","steps":["trace[200863265] 'agreement among raft nodes before linearized reading' (duration: 437.461193ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:49.702582Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:49.264709Z","time spent":"437.861048ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":465,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:49.702463Z","caller":"traceutil/trace.go:172","msg":"trace[1011812512] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1074; }","duration":"523.412529ms","start":"2026-03-30T16:49:49.179035Z","end":"2026-03-30T16:49:49.702448Z","steps":["trace[1011812512] 'agreement among raft nodes before linearized reading' (duration: 523.319658ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:49.702843Z","caller":"traceutil/trace.go:172","msg":"trace[1090131819] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1074; }","duration":"165.996347ms","start":"2026-03-30T16:49:49.536387Z","end":"2026-03-30T16:49:49.702383Z","steps":["trace[1090131819] 'agreement among raft nodes before linearized reading' (duration: 165.783874ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:49.702875Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"776.098679ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T16:49:49.703049Z","caller":"traceutil/trace.go:172","msg":"trace[305620700] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1074; }","duration":"776.287441ms","start":"2026-03-30T16:49:48.926747Z","end":"2026-03-30T16:49:49.703034Z","steps":["trace[305620700] 'agreement among raft nodes before linearized reading' (duration: 775.801626ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:49.703134Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:48.926725Z","time spent":"776.359631ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:50.553650Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"191.439902ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:49:51.406635Z","caller":"traceutil/trace.go:172","msg":"trace[947850592] linearizableReadLoop","detail":"{readStateIndex:1255; appliedIndex:1255; }","duration":"224.910885ms","start":"2026-03-30T16:49:51.181690Z","end":"2026-03-30T16:49:51.406601Z","steps":["trace[947850592] 'read index received' (duration: 224.897855ms)","trace[947850592] 'applied index is now lower than readState.Index' (duration: 10.87µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:51.406756Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"225.040176ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:51.406789Z","caller":"traceutil/trace.go:172","msg":"trace[233185576] transaction","detail":"{read_only:false; response_revision:1079; number_of_response:1; }","duration":"316.877724ms","start":"2026-03-30T16:49:51.089891Z","end":"2026-03-30T16:49:51.406769Z","steps":["trace[233185576] 'process raft request' (duration: 316.733062ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:51.406812Z","caller":"traceutil/trace.go:172","msg":"trace[381161842] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1078; }","duration":"225.120047ms","start":"2026-03-30T16:49:51.181684Z","end":"2026-03-30T16:49:51.406804Z","steps":["trace[381161842] 'agreement among raft nodes before linearized reading' (duration: 225.021476ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:51.406929Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:51.089868Z","time spent":"316.973475ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:49:52.037469Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004178,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:52.538218Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004178,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:53.038456Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004178,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:53.462639Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.971985347s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T16:49:53.462782Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.849271805s"} {"level":"warn","ts":"2026-03-30T16:49:53.462812Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.849307005s"} {"level":"info","ts":"2026-03-30T16:49:53.464032Z","caller":"traceutil/trace.go:172","msg":"trace[1088081538] linearizableReadLoop","detail":"{readStateIndex:1257; appliedIndex:1258; }","duration":"1.927478909s","start":"2026-03-30T16:49:51.536526Z","end":"2026-03-30T16:49:53.464005Z","steps":["trace[1088081538] 'read index received' (duration: 1.927466869s)","trace[1088081538] 'applied index is now lower than readState.Index' (duration: 6.06µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:53.464265Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.927717172s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:53.464627Z","caller":"traceutil/trace.go:172","msg":"trace[1271083094] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1081; }","duration":"1.928082867s","start":"2026-03-30T16:49:51.536522Z","end":"2026-03-30T16:49:53.464605Z","steps":["trace[1271083094] 'agreement among raft nodes before linearized reading' (duration: 1.927671852s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:53.464691Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:51.536503Z","time spent":"1.928173288s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:49:53.465547Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"183.331783ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"warn","ts":"2026-03-30T16:49:53.465607Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"223.738013ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csidrivers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:49:53.465641Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"972.932466ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/runtimeclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:53.465671Z","caller":"traceutil/trace.go:172","msg":"trace[363145067] range","detail":"{range_begin:/registry/csidrivers; range_end:; response_count:0; response_revision:1081; }","duration":"223.818124ms","start":"2026-03-30T16:49:53.241834Z","end":"2026-03-30T16:49:53.465652Z","steps":["trace[363145067] 'agreement among raft nodes before linearized reading' (duration: 223.690003ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:53.465697Z","caller":"traceutil/trace.go:172","msg":"trace[1298918129] range","detail":"{range_begin:/registry/runtimeclasses; range_end:; response_count:0; response_revision:1081; }","duration":"972.990586ms","start":"2026-03-30T16:49:52.492691Z","end":"2026-03-30T16:49:53.465682Z","steps":["trace[1298918129] 'agreement among raft nodes before linearized reading' (duration: 972.880315ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:53.465732Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:52.492667Z","time spent":"973.056728ms","remote":"127.0.0.1:55614","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/runtimeclasses\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:53.465752Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"125.260349ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T16:49:53.465793Z","caller":"traceutil/trace.go:172","msg":"trace[1114549888] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1081; }","duration":"125.30464ms","start":"2026-03-30T16:49:53.340478Z","end":"2026-03-30T16:49:53.465783Z","steps":["trace[1114549888] 'agreement among raft nodes before linearized reading' (duration: 125.187598ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:53.465800Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"711.712279ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T16:49:53.465627Z","caller":"traceutil/trace.go:172","msg":"trace[661549260] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1081; }","duration":"183.414184ms","start":"2026-03-30T16:49:53.282197Z","end":"2026-03-30T16:49:53.465612Z","steps":["trace[661549260] 'agreement among raft nodes before linearized reading' (duration: 183.183091ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:53.465943Z","caller":"traceutil/trace.go:172","msg":"trace[1271548441] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1081; }","duration":"711.81846ms","start":"2026-03-30T16:49:52.754069Z","end":"2026-03-30T16:49:53.465887Z","steps":["trace[1271548441] 'agreement among raft nodes before linearized reading' (duration: 711.400905ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:53.466036Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:52.754052Z","time spent":"711.929291ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:53.466063Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.287329595s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:53.466133Z","caller":"traceutil/trace.go:172","msg":"trace[1148378041] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1081; }","duration":"1.287402016s","start":"2026-03-30T16:49:52.178717Z","end":"2026-03-30T16:49:53.466119Z","steps":["trace[1148378041] 'agreement among raft nodes before linearized reading' (duration: 1.286641768s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:54.229500Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004183,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:54.308054Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"195.408241ms"} {"level":"warn","ts":"2026-03-30T16:49:54.308097Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"195.469282ms"} {"level":"info","ts":"2026-03-30T16:49:54.308136Z","caller":"traceutil/trace.go:172","msg":"trace[1708869046] linearizableReadLoop","detail":"{readStateIndex:1258; appliedIndex:1258; }","duration":"579.167659ms","start":"2026-03-30T16:49:53.728939Z","end":"2026-03-30T16:49:54.308107Z","steps":["trace[1708869046] 'read index received' (duration: 579.141599ms)","trace[1708869046] 'applied index is now lower than readState.Index' (duration: 10.68µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:54.308451Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"579.489542ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T16:49:54.308521Z","caller":"traceutil/trace.go:172","msg":"trace[987938540] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1081; }","duration":"579.579903ms","start":"2026-03-30T16:49:53.728924Z","end":"2026-03-30T16:49:54.308504Z","steps":["trace[987938540] 'agreement among raft nodes before linearized reading' (duration: 579.333941ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:54.308584Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:53.728899Z","time spent":"579.668654ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:54.576695Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"68.580428ms"} {"level":"warn","ts":"2026-03-30T16:49:54.576796Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"68.6902ms"} {"level":"warn","ts":"2026-03-30T16:49:54.576709Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004184,"received-request-id":10218277501228004183} {"level":"warn","ts":"2026-03-30T16:49:54.808631Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004184,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:54.850170Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"7.548523ms"} {"level":"warn","ts":"2026-03-30T16:49:54.850224Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"7.608824ms"} {"level":"info","ts":"2026-03-30T16:49:54.850211Z","caller":"traceutil/trace.go:172","msg":"trace[1484435638] linearizableReadLoop","detail":"{readStateIndex:1260; appliedIndex:1262; }","duration":"541.983219ms","start":"2026-03-30T16:49:54.308201Z","end":"2026-03-30T16:49:54.850185Z","steps":["trace[1484435638] 'read index received' (duration: 541.973919ms)","trace[1484435638] 'applied index is now lower than readState.Index' (duration: 7.89µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:54.906078Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"727.370851ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:54.906221Z","caller":"traceutil/trace.go:172","msg":"trace[286372268] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1085; }","duration":"727.530813ms","start":"2026-03-30T16:49:54.178650Z","end":"2026-03-30T16:49:54.906181Z","steps":["trace[286372268] 'agreement among raft nodes before linearized reading' (duration: 671.668064ms)","trace[286372268] 'range keys from in-memory index tree' (duration: 55.630316ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:49:54.906333Z","caller":"traceutil/trace.go:172","msg":"trace[1058927182] transaction","detail":"{read_only:false; response_revision:1087; number_of_response:1; }","duration":"295.121388ms","start":"2026-03-30T16:49:54.611188Z","end":"2026-03-30T16:49:54.906309Z","steps":["trace[1058927182] 'process raft request' (duration: 295.028117ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:54.906557Z","caller":"traceutil/trace.go:172","msg":"trace[1576269957] transaction","detail":"{read_only:false; response_revision:1086; number_of_response:1; }","duration":"592.761702ms","start":"2026-03-30T16:49:54.313777Z","end":"2026-03-30T16:49:54.906538Z","steps":["trace[1576269957] 'process raft request' (duration: 536.509719ms)","trace[1576269957] 'compare' (duration: 55.437425ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:54.906685Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:54.313756Z","time spent":"592.860803ms","remote":"127.0.0.1:56066","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":932,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:49:55.142125Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"91.719698ms"} {"level":"warn","ts":"2026-03-30T16:49:55.142188Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"91.790759ms"} {"level":"info","ts":"2026-03-30T16:49:55.142274Z","caller":"traceutil/trace.go:172","msg":"trace[390899275] linearizableReadLoop","detail":"{readStateIndex:1264; appliedIndex:1264; }","duration":"291.95401ms","start":"2026-03-30T16:49:54.850269Z","end":"2026-03-30T16:49:55.142223Z","steps":["trace[390899275] 'read index received' (duration: 291.93148ms)","trace[390899275] 'applied index is now lower than readState.Index' (duration: 19.6µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:55.142540Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"607.516381ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:55.142598Z","caller":"traceutil/trace.go:172","msg":"trace[372415284] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1087; }","duration":"607.604952ms","start":"2026-03-30T16:49:54.534978Z","end":"2026-03-30T16:49:55.142583Z","steps":["trace[372415284] 'agreement among raft nodes before linearized reading' (duration: 607.43981ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:55.142648Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:54.534955Z","time spent":"607.682973ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:49:55.301454Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:48.300156Z","time spent":"7.001289344s","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-30T16:49:55.644930Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004188,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:49:55.660670Z","caller":"traceutil/trace.go:172","msg":"trace[1713545767] transaction","detail":"{read_only:false; response_revision:1093; number_of_response:1; }","duration":"350.265733ms","start":"2026-03-30T16:49:55.310389Z","end":"2026-03-30T16:49:55.660655Z","steps":["trace[1713545767] 'process raft request' (duration: 345.609812ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:55.660758Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"516.260819ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:49:55.660792Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:55.310366Z","time spent":"350.364194ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":15664,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:55.660836Z","caller":"traceutil/trace.go:172","msg":"trace[1907479499] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1093; }","duration":"516.318059ms","start":"2026-03-30T16:49:55.144473Z","end":"2026-03-30T16:49:55.660791Z","steps":["trace[1907479499] 'agreement among raft nodes before linearized reading' (duration: 516.236749ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:49:55.660663Z","caller":"traceutil/trace.go:172","msg":"trace[891942591] linearizableReadLoop","detail":"{readStateIndex:1273; appliedIndex:1272; }","duration":"516.160068ms","start":"2026-03-30T16:49:55.144481Z","end":"2026-03-30T16:49:55.660641Z","steps":["trace[891942591] 'read index received' (duration: 511.266015ms)","trace[891942591] 'applied index is now lower than readState.Index' (duration: 4.893113ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:55.660873Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:55.144449Z","time spent":"516.416611ms","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:49:55.726087Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"215.423515ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.227\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-30T16:49:55.726215Z","caller":"traceutil/trace.go:172","msg":"trace[593582177] range","detail":"{range_begin:/registry/masterleases/199.204.45.227; range_end:; response_count:1; response_revision:1102; }","duration":"215.560086ms","start":"2026-03-30T16:49:55.510644Z","end":"2026-03-30T16:49:55.726204Z","steps":["trace[593582177] 'agreement among raft nodes before linearized reading' (duration: 215.353304ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:55.726211Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"547.046524ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:55.726324Z","caller":"traceutil/trace.go:172","msg":"trace[243753552] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1102; }","duration":"547.188775ms","start":"2026-03-30T16:49:55.179127Z","end":"2026-03-30T16:49:55.726316Z","steps":["trace[243753552] 'agreement among raft nodes before linearized reading' (duration: 547.007843ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:55.787204Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"115.71849ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:55.787266Z","caller":"traceutil/trace.go:172","msg":"trace[1386722980] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1102; }","duration":"115.790461ms","start":"2026-03-30T16:49:55.671463Z","end":"2026-03-30T16:49:55.787253Z","steps":["trace[1386722980] 'agreement among raft nodes before linearized reading' (duration: 102.009131ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:56.119382Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"118.010228ms"} {"level":"warn","ts":"2026-03-30T16:49:56.119456Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"118.091149ms"} {"level":"warn","ts":"2026-03-30T16:49:56.284490Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004196,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:56.450676Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"271.634631ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:56.450732Z","caller":"traceutil/trace.go:172","msg":"trace[1459334085] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1104; }","duration":"271.702792ms","start":"2026-03-30T16:49:56.179018Z","end":"2026-03-30T16:49:56.450721Z","steps":["trace[1459334085] 'range keys from in-memory index tree' (duration: 271.58726ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:56.450968Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"331.233179ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:5"} {"level":"warn","ts":"2026-03-30T16:49:56.452448Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"133.064129ms"} {"level":"warn","ts":"2026-03-30T16:49:56.452513Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"133.057459ms"} {"level":"info","ts":"2026-03-30T16:49:56.452895Z","caller":"traceutil/trace.go:172","msg":"trace[379436013] linearizableReadLoop","detail":"{readStateIndex:1293; appliedIndex:1290; }","duration":"668.489386ms","start":"2026-03-30T16:49:55.784398Z","end":"2026-03-30T16:49:56.452888Z","steps":["trace[379436013] 'read index received' (duration: 668.275024ms)","trace[379436013] 'applied index is now lower than readState.Index' (duration: 214.082µs)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:49:56.452974Z","caller":"traceutil/trace.go:172","msg":"trace[804605505] transaction","detail":"{read_only:false; response_revision:1106; number_of_response:1; }","duration":"664.633025ms","start":"2026-03-30T16:49:55.788335Z","end":"2026-03-30T16:49:56.452968Z","steps":["trace[804605505] 'process raft request' (duration: 664.519634ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:56.453016Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:55.788323Z","time spent":"664.664575ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:49:56.453738Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"669.337036ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-30T16:49:56.453796Z","caller":"traceutil/trace.go:172","msg":"trace[693436647] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1106; }","duration":"669.395046ms","start":"2026-03-30T16:49:55.784393Z","end":"2026-03-30T16:49:56.453788Z","steps":["trace[693436647] 'agreement among raft nodes before linearized reading' (duration: 668.799819ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:56.453829Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:55.784375Z","time spent":"669.448397ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"info","ts":"2026-03-30T16:49:56.604324Z","caller":"traceutil/trace.go:172","msg":"trace[1172663603] linearizableReadLoop","detail":"{readStateIndex:1293; appliedIndex:1293; }","duration":"151.342995ms","start":"2026-03-30T16:49:56.452922Z","end":"2026-03-30T16:49:56.604265Z","steps":["trace[1172663603] 'read index received' (duration: 151.331955ms)","trace[1172663603] 'applied index is now lower than readState.Index' (duration: 9.37µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:56.653597Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"202.811453ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:56.653680Z","caller":"traceutil/trace.go:172","msg":"trace[1770991185] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1106; }","duration":"202.906305ms","start":"2026-03-30T16:49:56.450754Z","end":"2026-03-30T16:49:56.653661Z","steps":["trace[1770991185] 'agreement among raft nodes before linearized reading' (duration: 153.69686ms)","trace[1770991185] 'range keys from in-memory index tree' (duration: 49.091634ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:56.654234Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"631.479271ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:103 size:8866"} {"level":"info","ts":"2026-03-30T16:49:56.654304Z","caller":"traceutil/trace.go:172","msg":"trace[1562348097] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:103; response_revision:1106; }","duration":"631.561401ms","start":"2026-03-30T16:49:56.022724Z","end":"2026-03-30T16:49:56.654286Z","steps":["trace[1562348097] 'agreement among raft nodes before linearized reading' (duration: 581.73839ms)","trace[1562348097] 'range keys from in-memory index tree' (duration: 49.220805ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:56.654338Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:56.022697Z","time spent":"631.631592ms","remote":"127.0.0.1:55972","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":103,"response size":8889,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"info","ts":"2026-03-30T16:49:56.654565Z","caller":"traceutil/trace.go:172","msg":"trace[866173572] transaction","detail":"{read_only:false; response_revision:1107; number_of_response:1; }","duration":"832.607719ms","start":"2026-03-30T16:49:55.821931Z","end":"2026-03-30T16:49:56.654539Z","steps":["trace[866173572] 'process raft request' (duration: 783.719718ms)","trace[866173572] 'compare' (duration: 47.82607ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:56.654724Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:55.821902Z","time spent":"832.742491ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":13205,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:56.655100Z","caller":"traceutil/trace.go:172","msg":"trace[1011157999] transaction","detail":"{read_only:false; response_revision:1108; number_of_response:1; }","duration":"799.976444ms","start":"2026-03-30T16:49:55.854606Z","end":"2026-03-30T16:49:56.654582Z","steps":["trace[1011157999] 'process raft request' (duration: 799.62206ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:56.655268Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:55.854583Z","time spent":"800.605311ms","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":208470,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:49:56.718515Z","caller":"traceutil/trace.go:172","msg":"trace[1441955999] linearizableReadLoop","detail":"{readStateIndex:1296; appliedIndex:1297; }","duration":"114.08916ms","start":"2026-03-30T16:49:56.604403Z","end":"2026-03-30T16:49:56.718492Z","steps":["trace[1441955999] 'read index received' (duration: 114.07612ms)","trace[1441955999] 'applied index is now lower than readState.Index' (duration: 7.97µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:56.718798Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"263.092419ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:460"} {"level":"info","ts":"2026-03-30T16:49:56.718842Z","caller":"traceutil/trace.go:172","msg":"trace[1367446803] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1108; }","duration":"263.154299ms","start":"2026-03-30T16:49:56.455677Z","end":"2026-03-30T16:49:56.718831Z","steps":["trace[1367446803] 'agreement among raft nodes before linearized reading' (duration: 262.910667ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:56.719026Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"184.789778ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:56.719181Z","caller":"traceutil/trace.go:172","msg":"trace[6149790] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1109; }","duration":"184.95192ms","start":"2026-03-30T16:49:56.534209Z","end":"2026-03-30T16:49:56.719161Z","steps":["trace[6149790] 'agreement among raft nodes before linearized reading' (duration: 184.668367ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:56.719267Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"225.57532ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T16:49:56.719343Z","caller":"traceutil/trace.go:172","msg":"trace[2060511670] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1110; }","duration":"225.661821ms","start":"2026-03-30T16:49:56.493665Z","end":"2026-03-30T16:49:56.719327Z","steps":["trace[2060511670] 'agreement among raft nodes before linearized reading' (duration: 225.113975ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:58.944377Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"331.03939ms"} {"level":"warn","ts":"2026-03-30T16:49:58.944438Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"331.105641ms"} {"level":"warn","ts":"2026-03-30T16:49:59.034787Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004238,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:49:59.186895Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"42.427701ms"} {"level":"info","ts":"2026-03-30T16:49:59.186989Z","caller":"traceutil/trace.go:172","msg":"trace[1268994782] linearizableReadLoop","detail":"{readStateIndex:1355; appliedIndex:1356; }","duration":"652.946377ms","start":"2026-03-30T16:49:58.534021Z","end":"2026-03-30T16:49:59.186968Z","steps":["trace[1268994782] 'read index received' (duration: 652.933227ms)","trace[1268994782] 'applied index is now lower than readState.Index' (duration: 11.8µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:59.187009Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"42.548703ms"} {"level":"warn","ts":"2026-03-30T16:49:59.250198Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"716.16736ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:49:59.250311Z","caller":"traceutil/trace.go:172","msg":"trace[1646357566] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1148; }","duration":"716.270311ms","start":"2026-03-30T16:49:58.534002Z","end":"2026-03-30T16:49:59.250273Z","steps":["trace[1646357566] 'agreement among raft nodes before linearized reading' (duration: 653.088688ms)","trace[1646357566] 'range keys from in-memory index tree' (duration: 63.038811ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:49:59.250377Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:58.533974Z","time spent":"716.388782ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:49:59.258152Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004240,"received-request-id":10218277501228004238} {"level":"info","ts":"2026-03-30T16:49:59.258591Z","caller":"traceutil/trace.go:172","msg":"trace[780701594] transaction","detail":"{read_only:false; response_revision:1149; number_of_response:1; }","duration":"741.248752ms","start":"2026-03-30T16:49:58.517321Z","end":"2026-03-30T16:49:59.258569Z","steps":["trace[780701594] 'process raft request' (duration: 740.963289ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:59.258631Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"637.64811ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:18354"} {"level":"info","ts":"2026-03-30T16:49:59.258688Z","caller":"traceutil/trace.go:172","msg":"trace[1789540294] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:1149; }","duration":"637.70924ms","start":"2026-03-30T16:49:58.620963Z","end":"2026-03-30T16:49:59.258673Z","steps":["trace[1789540294] 'agreement among raft nodes before linearized reading' (duration: 637.526608ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:49:59.258727Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:58.620938Z","time spent":"637.778681ms","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18377,"request content":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:49:59.258734Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:49:58.517294Z","time spent":"741.374783ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:00.342726Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"325.289532ms"} {"level":"warn","ts":"2026-03-30T16:50:00.342827Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"325.397683ms"} {"level":"info","ts":"2026-03-30T16:50:00.495563Z","caller":"traceutil/trace.go:172","msg":"trace[1293513994] linearizableReadLoop","detail":"{readStateIndex:1365; appliedIndex:1366; }","duration":"317.113882ms","start":"2026-03-30T16:50:00.178422Z","end":"2026-03-30T16:50:00.495536Z","steps":["trace[1293513994] 'read index received' (duration: 317.105692ms)","trace[1293513994] 'applied index is now lower than readState.Index' (duration: 4.12µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:00.536078Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"357.648601ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:00.536163Z","caller":"traceutil/trace.go:172","msg":"trace[1392015680] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1158; }","duration":"357.756262ms","start":"2026-03-30T16:50:00.178386Z","end":"2026-03-30T16:50:00.536143Z","steps":["trace[1392015680] 'agreement among raft nodes before linearized reading' (duration: 317.312735ms)","trace[1392015680] 'range keys from in-memory index tree' (duration: 40.315706ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:00.536564Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"208.593181ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/certificatesigningrequests\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:00.536601Z","caller":"traceutil/trace.go:172","msg":"trace[1953096746] range","detail":"{range_begin:/registry/certificatesigningrequests; range_end:; response_count:0; response_revision:1159; }","duration":"208.630282ms","start":"2026-03-30T16:50:00.327962Z","end":"2026-03-30T16:50:00.536592Z","steps":["trace[1953096746] 'agreement among raft nodes before linearized reading' (duration: 208.567801ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:00.536961Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"178.063542ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T16:50:00.537553Z","caller":"traceutil/trace.go:172","msg":"trace[1605120575] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1159; }","duration":"178.937171ms","start":"2026-03-30T16:50:00.358592Z","end":"2026-03-30T16:50:00.537529Z","steps":["trace[1605120575] 'agreement among raft nodes before linearized reading' (duration: 177.90186ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:01.155945Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"42.584625ms"} {"level":"warn","ts":"2026-03-30T16:50:01.156027Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"42.678206ms"} {"level":"warn","ts":"2026-03-30T16:50:01.535148Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004250,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:02.035920Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004250,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:50:02.458102Z","caller":"traceutil/trace.go:172","msg":"trace[1587884655] linearizableReadLoop","detail":"{readStateIndex:1367; appliedIndex:1368; }","duration":"1.423094676s","start":"2026-03-30T16:50:01.034982Z","end":"2026-03-30T16:50:02.458077Z","steps":["trace[1587884655] 'read index received' (duration: 1.423085916s)","trace[1587884655] 'applied index is now lower than readState.Index' (duration: 7.35µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:02.458320Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.423324799s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/mutatingwebhookconfigurations\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:02.458328Z","caller":"traceutil/trace.go:172","msg":"trace[1314066111] transaction","detail":"{read_only:false; response_revision:1161; number_of_response:1; }","duration":"976.99528ms","start":"2026-03-30T16:50:01.481318Z","end":"2026-03-30T16:50:02.458313Z","steps":["trace[1314066111] 'process raft request' (duration: 976.861188ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:02.458493Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:01.481294Z","time spent":"977.135201ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:50:02.458363Z","caller":"traceutil/trace.go:172","msg":"trace[1496548130] range","detail":"{range_begin:/registry/mutatingwebhookconfigurations; range_end:; response_count:0; response_revision:1160; }","duration":"1.423386379s","start":"2026-03-30T16:50:01.034965Z","end":"2026-03-30T16:50:02.458352Z","steps":["trace[1496548130] 'agreement among raft nodes before linearized reading' (duration: 1.423275658s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:02.458635Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:01.034941Z","time spent":"1.423657742s","remote":"127.0.0.1:55922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":0,"response size":28,"request content":"key:\"/registry/mutatingwebhookconfigurations\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:02.459522Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"331.726469ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ipaddresses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:02.459516Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.280743073s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:02.459564Z","caller":"traceutil/trace.go:172","msg":"trace[988116828] range","detail":"{range_begin:/registry/ipaddresses; range_end:; response_count:0; response_revision:1163; }","duration":"331.77376ms","start":"2026-03-30T16:50:02.127782Z","end":"2026-03-30T16:50:02.459556Z","steps":["trace[988116828] 'agreement among raft nodes before linearized reading' (duration: 331.699658ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:02.459581Z","caller":"traceutil/trace.go:172","msg":"trace[1083054045] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1163; }","duration":"1.280815774s","start":"2026-03-30T16:50:01.178751Z","end":"2026-03-30T16:50:02.459566Z","steps":["trace[1083054045] 'agreement among raft nodes before linearized reading' (duration: 1.280727442s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:02.459589Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:02.127764Z","time spent":"331.818061ms","remote":"127.0.0.1:55584","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/ipaddresses\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:02.459631Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"923.500793ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:02.459718Z","caller":"traceutil/trace.go:172","msg":"trace[648794704] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1163; }","duration":"923.570224ms","start":"2026-03-30T16:50:01.536115Z","end":"2026-03-30T16:50:02.459685Z","steps":["trace[648794704] 'agreement among raft nodes before linearized reading' (duration: 923.443823ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:02.459766Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:01.536101Z","time spent":"923.653905ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:02.912602Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"199.670068ms"} {"level":"warn","ts":"2026-03-30T16:50:02.912781Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"199.86078ms"} {"level":"info","ts":"2026-03-30T16:50:02.939637Z","caller":"traceutil/trace.go:172","msg":"trace[1910676737] linearizableReadLoop","detail":"{readStateIndex:1372; appliedIndex:1373; }","duration":"403.33585ms","start":"2026-03-30T16:50:02.536266Z","end":"2026-03-30T16:50:02.939601Z","steps":["trace[1910676737] 'read index received' (duration: 403.327109ms)","trace[1910676737] 'applied index is now lower than readState.Index' (duration: 6.92µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:02.939855Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"403.574132ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:02.939896Z","caller":"traceutil/trace.go:172","msg":"trace[487455350] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1165; }","duration":"403.628822ms","start":"2026-03-30T16:50:02.536257Z","end":"2026-03-30T16:50:02.939885Z","steps":["trace[487455350] 'agreement among raft nodes before linearized reading' (duration: 403.532251ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:02.939930Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:02.536241Z","time spent":"403.682033ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:02.958060Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"417.578542ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:109 size:9346"} {"level":"info","ts":"2026-03-30T16:50:02.958120Z","caller":"traceutil/trace.go:172","msg":"trace[1437966544] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:109; response_revision:1165; }","duration":"417.655113ms","start":"2026-03-30T16:50:02.540455Z","end":"2026-03-30T16:50:02.958110Z","steps":["trace[1437966544] 'agreement among raft nodes before linearized reading' (duration: 417.157858ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:02.958147Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:02.540447Z","time spent":"417.693664ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":109,"response size":9369,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"info","ts":"2026-03-30T16:50:03.210255Z","caller":"traceutil/trace.go:172","msg":"trace[1006608816] linearizableReadLoop","detail":"{readStateIndex:1374; appliedIndex:1374; }","duration":"146.570494ms","start":"2026-03-30T16:50:03.063642Z","end":"2026-03-30T16:50:03.210212Z","steps":["trace[1006608816] 'read index received' (duration: 146.559233ms)","trace[1006608816] 'applied index is now lower than readState.Index' (duration: 10.011µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:03.210695Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"147.03628ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T16:50:03.210778Z","caller":"traceutil/trace.go:172","msg":"trace[1441741000] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1167; }","duration":"147.125751ms","start":"2026-03-30T16:50:03.063626Z","end":"2026-03-30T16:50:03.210751Z","steps":["trace[1441741000] 'agreement among raft nodes before linearized reading' (duration: 146.820198ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:03.666493Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"152.995092ms"} {"level":"warn","ts":"2026-03-30T16:50:03.666613Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"153.128253ms"} {"level":"info","ts":"2026-03-30T16:50:03.735830Z","caller":"traceutil/trace.go:172","msg":"trace[338570756] linearizableReadLoop","detail":"{readStateIndex:1375; appliedIndex:1375; }","duration":"247.282757ms","start":"2026-03-30T16:50:03.488516Z","end":"2026-03-30T16:50:03.735798Z","steps":["trace[338570756] 'read index received' (duration: 247.275687ms)","trace[338570756] 'applied index is now lower than readState.Index' (duration: 6.37µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:03.736134Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"247.59691ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:50:03.736166Z","caller":"traceutil/trace.go:172","msg":"trace[1770485794] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1167; }","duration":"247.64892ms","start":"2026-03-30T16:50:03.488509Z","end":"2026-03-30T16:50:03.736158Z","steps":["trace[1770485794] 'agreement among raft nodes before linearized reading' (duration: 247.439328ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:03.738169Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"201.492334ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:03.738244Z","caller":"traceutil/trace.go:172","msg":"trace[378434460] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1168; }","duration":"201.571695ms","start":"2026-03-30T16:50:03.536655Z","end":"2026-03-30T16:50:03.738226Z","steps":["trace[378434460] 'agreement among raft nodes before linearized reading' (duration: 201.462564ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:04.504075Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004262,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:05.005186Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004262,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:05.248395Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.069069061s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:05.248663Z","caller":"traceutil/trace.go:172","msg":"trace[1627742127] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1169; }","duration":"1.069352164s","start":"2026-03-30T16:50:04.179295Z","end":"2026-03-30T16:50:05.248648Z","steps":["trace[1627742127] 'range keys from in-memory index tree' (duration: 1.069012021s)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:05.314829Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"a8d4e8c653fa8dce [logterm: 3, index: 1381, vote: a8d4e8c653fa8dce] ignored MsgPreVote from 3cf904ec39d3e333 [logterm: 3, index: 1378] at term 3: lease is not expired (remaining ticks: 3)"} {"level":"warn","ts":"2026-03-30T16:50:05.505529Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004262,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:05.592454Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.599546334s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T16:50:05.592596Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.479087967s"} {"level":"warn","ts":"2026-03-30T16:50:05.592624Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.479119208s"} {"level":"warn","ts":"2026-03-30T16:50:06.006022Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004262,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:06.230544Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"437.889577ms"} {"level":"warn","ts":"2026-03-30T16:50:06.230608Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"437.961318ms"} {"level":"info","ts":"2026-03-30T16:50:06.230600Z","caller":"traceutil/trace.go:172","msg":"trace[1899797280] linearizableReadLoop","detail":"{readStateIndex:1377; appliedIndex:1378; }","duration":"2.227308761s","start":"2026-03-30T16:50:04.003260Z","end":"2026-03-30T16:50:06.230569Z","steps":["trace[1899797280] 'read index received' (duration: 2.227294621s)","trace[1899797280] 'applied index is now lower than readState.Index' (duration: 12.72µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:06.515098Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.511807573s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:50:06.515159Z","caller":"traceutil/trace.go:172","msg":"trace[1622332535] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1170; }","duration":"2.511889704s","start":"2026-03-30T16:50:04.003256Z","end":"2026-03-30T16:50:06.515146Z","steps":["trace[1622332535] 'agreement among raft nodes before linearized reading' (duration: 2.227479473s)","trace[1622332535] 'range keys from in-memory index tree' (duration: 284.236199ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:06.515185Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:04.003242Z","time spent":"2.511937324s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":465,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:06.515529Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"284.796925ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T16:50:06.640439Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"209.804732ms"} {"level":"warn","ts":"2026-03-30T16:50:06.640501Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"209.875062ms"} {"level":"warn","ts":"2026-03-30T16:50:06.640597Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004269,"received-request-id":10218277501228004262} {"level":"info","ts":"2026-03-30T16:50:06.642191Z","caller":"traceutil/trace.go:172","msg":"trace[1591782914] linearizableReadLoop","detail":"{readStateIndex:1381; appliedIndex:1381; }","duration":"411.502461ms","start":"2026-03-30T16:50:06.230667Z","end":"2026-03-30T16:50:06.642169Z","steps":["trace[1591782914] 'read index received' (duration: 411.495321ms)","trace[1591782914] 'applied index is now lower than readState.Index' (duration: 5.96µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:06.803059Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.5543234s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:06.803204Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.2667589s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:06.803218Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.0341079s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T16:50:06.803256Z","caller":"traceutil/trace.go:172","msg":"trace[163160995] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1173; }","duration":"2.03414371s","start":"2026-03-30T16:50:04.769101Z","end":"2026-03-30T16:50:06.803245Z","steps":["trace[163160995] 'agreement among raft nodes before linearized reading' (duration: 1.873184641s)","trace[163160995] 'range keys from in-memory index tree' (duration: 160.902319ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:06.803298Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:04.769079Z","time spent":"2.034209791s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":506,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:06.803339Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.29189843s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.227\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"warn","ts":"2026-03-30T16:50:06.803443Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"162.741969ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:06.803449Z","caller":"traceutil/trace.go:172","msg":"trace[2067565307] range","detail":"{range_begin:/registry/masterleases/199.204.45.227; range_end:; response_count:1; response_revision:1173; }","duration":"1.292029391s","start":"2026-03-30T16:50:05.511394Z","end":"2026-03-30T16:50:06.803424Z","steps":["trace[2067565307] 'agreement among raft nodes before linearized reading' (duration: 1.13089358s)","trace[2067565307] 'range keys from in-memory index tree' (duration: 160.824278ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:06.803525Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:05.511366Z","time spent":"1.292147252s","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.227\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:06.803257Z","caller":"traceutil/trace.go:172","msg":"trace[1462041107] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1173; }","duration":"1.266814431s","start":"2026-03-30T16:50:05.536427Z","end":"2026-03-30T16:50:06.803241Z","steps":["trace[1462041107] 'agreement among raft nodes before linearized reading' (duration: 1.105800151s)","trace[1462041107] 'range keys from in-memory index tree' (duration: 160.918388ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:06.803359Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.993524035s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces/kube-system\" limit:1 ","response":"range_response_count:1 size:351"} {"level":"info","ts":"2026-03-30T16:50:06.803738Z","caller":"traceutil/trace.go:172","msg":"trace[1078509783] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1173; }","duration":"1.554430971s","start":"2026-03-30T16:50:05.248699Z","end":"2026-03-30T16:50:06.803130Z","steps":["trace[1078509783] 'agreement among raft nodes before linearized reading' (duration: 1.393578833s)","trace[1078509783] 'range keys from in-memory index tree' (duration: 160.718377ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:06.803762Z","caller":"traceutil/trace.go:172","msg":"trace[1739320206] range","detail":"{range_begin:/registry/namespaces/kube-system; range_end:; response_count:1; response_revision:1173; }","duration":"1.993912399s","start":"2026-03-30T16:50:04.809811Z","end":"2026-03-30T16:50:06.803723Z","steps":["trace[1739320206] 'agreement among raft nodes before linearized reading' (duration: 1.832460205s)","trace[1739320206] 'range keys from in-memory index tree' (duration: 160.955879ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:06.803643Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:05.536409Z","time spent":"1.267220095s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:06.803861Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:04.809793Z","time spent":"1.994029811s","remote":"127.0.0.1:55234","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":1,"response size":374,"request content":"key:\"/registry/namespaces/kube-system\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:07.922645Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.195250696s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T16:50:07.922783Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.009097824s"} {"level":"warn","ts":"2026-03-30T16:50:07.922809Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.009121704s"} {"level":"warn","ts":"2026-03-30T16:50:08.804886Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.998854984s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T16:50:08.804976Z","caller":"traceutil/trace.go:172","msg":"trace[1486939593] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.998959865s","start":"2026-03-30T16:50:06.805997Z","end":"2026-03-30T16:50:08.804957Z","steps":["trace[1486939593] 'agreement among raft nodes before linearized reading' (duration: 1.998849124s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:08.805040Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:06.805977Z","time spent":"1.999052466s","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 16:50:08 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:08.830339Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999788173s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T16:50:08.830429Z","caller":"traceutil/trace.go:172","msg":"trace[467509472] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.999884034s","start":"2026-03-30T16:50:06.830520Z","end":"2026-03-30T16:50:08.830404Z","steps":["trace[467509472] 'agreement among raft nodes before linearized reading' (duration: 1.999783733s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:08.830502Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:06.830512Z","time spent":"1.999981365s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 16:50:08 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:10.078781Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"2.154451615s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T16:50:10.078976Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.956136692s"} {"level":"warn","ts":"2026-03-30T16:50:10.079033Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.956209172s"} {"level":"warn","ts":"2026-03-30T16:50:10.448827Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.64533181s","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-30T16:50:10.448994Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.643529351s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:10.449041Z","caller":"traceutil/trace.go:172","msg":"trace[1615200558] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1174; }","duration":"3.643592721s","start":"2026-03-30T16:50:06.805435Z","end":"2026-03-30T16:50:10.449027Z","steps":["trace[1615200558] 'range keys from in-memory index tree' (duration: 3.64348949s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:10.468914Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"189.853376ms"} {"level":"warn","ts":"2026-03-30T16:50:10.468976Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"189.922947ms"} {"level":"warn","ts":"2026-03-30T16:50:10.814707Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.00061094s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-30T16:50:10.815022Z","caller":"traceutil/trace.go:172","msg":"trace[1420178342] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000936213s","start":"2026-03-30T16:50:08.814062Z","end":"2026-03-30T16:50:10.814999Z","steps":["trace[1420178342] 'agreement among raft nodes before linearized reading' (duration: 2.000603819s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:10.815238Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:08.814046Z","time spent":"2.001102775s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 16:50:10 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:11.391094Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"650.780819ms"} {"level":"warn","ts":"2026-03-30T16:50:11.391160Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"650.85543ms"} {"level":"warn","ts":"2026-03-30T16:50:11.471497Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.99975963s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T16:50:11.471565Z","caller":"traceutil/trace.go:172","msg":"trace[343512468] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"4.999862881s","start":"2026-03-30T16:50:06.471688Z","end":"2026-03-30T16:50:11.471551Z","steps":["trace[343512468] 'agreement among raft nodes before linearized reading' (duration: 4.99975628s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:11.471615Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:06.471667Z","time spent":"4.999937872s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} 2026/03/30 16:50:11 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:12.815286Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000036813s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T16:50:12.815432Z","caller":"traceutil/trace.go:172","msg":"trace[896295691] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000199145s","start":"2026-03-30T16:50:10.815212Z","end":"2026-03-30T16:50:12.815411Z","steps":["trace[896295691] 'agreement among raft nodes before linearized reading' (duration: 2.000029583s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:12.815621Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:10.815184Z","time spent":"2.000423657s","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 16:50:12 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:12.836313Z","caller":"etcdserver/server.go:900","msg":"failed to revoke lease","lease-id":"0dce9d3fa50fb31f","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-30T16:50:13.483537Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"a8d4e8c653fa8dce [logterm: 3, index: 1414, vote: a8d4e8c653fa8dce] ignored MsgPreVote from c3874f4929b5956e [logterm: 3, index: 1410] at term 3: lease is not expired (remaining ticks: 4)"} {"level":"warn","ts":"2026-03-30T16:50:13.806813Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:06.805649Z","time spent":"7.001157864s","remote":"127.0.0.1:55086","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-30T16:50:14.014696Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"a8d4e8c653fa8dce [logterm: 3, index: 1417, vote: a8d4e8c653fa8dce] ignored MsgPreVote from 3cf904ec39d3e333 [logterm: 3, index: 1410] at term 3: lease is not expired (remaining ticks: 4)"} {"level":"info","ts":"2026-03-30T16:50:14.320663Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"a8d4e8c653fa8dce [logterm: 3, index: 1417, vote: a8d4e8c653fa8dce] ignored MsgVote from 3cf904ec39d3e333 [logterm: 3, index: 1410] at term 3: lease is not expired (remaining ticks: 4)"} {"level":"info","ts":"2026-03-30T16:50:14.510364Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"a8d4e8c653fa8dce [term: 3] received a MsgApp message with higher term from 3cf904ec39d3e333 [term: 4]"} {"level":"info","ts":"2026-03-30T16:50:14.510489Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"a8d4e8c653fa8dce became follower at term 4"} {"level":"info","ts":"2026-03-30T16:50:14.510522Z","logger":"raft","caller":"v3@v3.6.0/log.go:158","msg":"found conflict at index 1411 [existing term: 3, conflicting term: 4]"} {"level":"info","ts":"2026-03-30T16:50:14.510543Z","logger":"raft","caller":"v3@v3.6.0/log_unstable.go:211","msg":"truncate the unstable entries before index 1411"} {"level":"info","ts":"2026-03-30T16:50:14.510766Z","logger":"raft","caller":"v3@v3.6.0/node.go:372","msg":"raft.node: a8d4e8c653fa8dce changed leader from a8d4e8c653fa8dce to 3cf904ec39d3e333 at term 4"} {"level":"warn","ts":"2026-03-30T16:50:14.822664Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.0002837s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T16:50:14.822850Z","caller":"traceutil/trace.go:172","msg":"trace[1961826321] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000487852s","start":"2026-03-30T16:50:12.822345Z","end":"2026-03-30T16:50:14.822833Z","steps":["trace[1961826321] 'agreement among raft nodes before linearized reading' (duration: 2.00028107s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:14.822951Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:12.822322Z","time spent":"2.000617063s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 16:50:14 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:14.971813Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999811431s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T16:50:14.971869Z","caller":"traceutil/trace.go:172","msg":"trace[505376672] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999883962s","start":"2026-03-30T16:50:09.971973Z","end":"2026-03-30T16:50:14.971857Z","steps":["trace[505376672] 'agreement among raft nodes before linearized reading' (duration: 4.999809361s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:14.971935Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:09.971950Z","time spent":"4.999960232s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} 2026/03/30 16:50:14 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:15.211435Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"3.420506035s","expected-duration":"1s"} {"level":"info","ts":"2026-03-30T16:50:15.211530Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.211571Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.211613Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.211623Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.211631Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.211640Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1164","msg":"a8d4e8c653fa8dce [term: 4] ignored entry appends from a MsgStorageAppendResp message with lower term [term: 3]"} {"level":"warn","ts":"2026-03-30T16:50:15.215727Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.604848595s","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T16:50:15.217653Z","caller":"traceutil/trace.go:172","msg":"trace[254962858] linearizableReadLoop","detail":"{readStateIndex:1389; appliedIndex:1381; }","duration":"8.5753879s","start":"2026-03-30T16:50:06.642242Z","end":"2026-03-30T16:50:15.217630Z","steps":["trace[254962858] 'read index received' (duration: 788.449µs)","trace[254962858] 'applied index is now lower than readState.Index' (duration: 8.574597041s)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:15.224078Z","caller":"traceutil/trace.go:172","msg":"trace[1019236491] transaction","detail":"{read_only:false; response_revision:1181; number_of_response:1; }","duration":"6.415018539s","start":"2026-03-30T16:50:08.809035Z","end":"2026-03-30T16:50:15.224053Z","steps":["trace[1019236491] 'process raft request' (duration: 6.414910778s)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:15.224075Z","caller":"traceutil/trace.go:172","msg":"trace[1515036140] transaction","detail":"{read_only:false; response_revision:1182; number_of_response:1; }","duration":"5.748924464s","start":"2026-03-30T16:50:09.475128Z","end":"2026-03-30T16:50:15.224052Z","steps":["trace[1515036140] 'process raft request' (duration: 5.748850884s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.224177Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:08.808981Z","time spent":"6.415149491s","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:15.224197Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:09.475106Z","time spent":"5.749033826s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:50:15.230943Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.231000Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.231011Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.231018Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.231025Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.231031Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.231038Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"a8d4e8c653fa8dce [term: 4] ignored a MsgAppResp message with lower term from a8d4e8c653fa8dce [term: 3]"} {"level":"info","ts":"2026-03-30T16:50:15.235335Z","caller":"etcdserver/v3_server.go:903","msg":"first commit in current term: resending ReadIndex request"} {"level":"warn","ts":"2026-03-30T16:50:15.252149Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.568815238s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/deployments\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:15.252234Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"429.215385ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:15.252236Z","caller":"traceutil/trace.go:172","msg":"trace[459009474] range","detail":"{range_begin:/registry/deployments; range_end:; response_count:0; response_revision:1195; }","duration":"3.568913949s","start":"2026-03-30T16:50:11.683305Z","end":"2026-03-30T16:50:15.252219Z","steps":["trace[459009474] 'agreement among raft nodes before linearized reading' (duration: 3.568763747s)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:15.252264Z","caller":"traceutil/trace.go:172","msg":"trace[719583952] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1195; }","duration":"429.241115ms","start":"2026-03-30T16:50:14.823012Z","end":"2026-03-30T16:50:15.252253Z","steps":["trace[719583952] 'agreement among raft nodes before linearized reading' (duration: 429.177614ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.252279Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:11.683284Z","time spent":"3.56898596s","remote":"127.0.0.1:55828","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/deployments\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.252299Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:14.822996Z","time spent":"429.296845ms","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:15.252288Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.274443203s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterroles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:15.252378Z","caller":"traceutil/trace.go:172","msg":"trace[2088943188] range","detail":"{range_begin:/registry/clusterroles; range_end:; response_count:0; response_revision:1195; }","duration":"3.274559634s","start":"2026-03-30T16:50:11.977800Z","end":"2026-03-30T16:50:15.252360Z","steps":["trace[2088943188] 'agreement among raft nodes before linearized reading' (duration: 3.274281831s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.252426Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:11.977780Z","time spent":"3.274635945s","remote":"127.0.0.1:55660","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/clusterroles\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.252500Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.365485666s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:15.252546Z","caller":"traceutil/trace.go:172","msg":"trace[1365575855] range","detail":"{range_begin:/registry/secrets; range_end:; response_count:0; response_revision:1195; }","duration":"4.365526927s","start":"2026-03-30T16:50:10.887006Z","end":"2026-03-30T16:50:15.252533Z","steps":["trace[1365575855] 'agreement among raft nodes before linearized reading' (duration: 4.365467386s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.252575Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:10.886970Z","time spent":"4.365597188s","remote":"127.0.0.1:55208","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":28,"request content":"key:\"/registry/secrets\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.252666Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.303063466s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:15.252696Z","caller":"traceutil/trace.go:172","msg":"trace[1103635261] range","detail":"{range_begin:/registry/resourceclaims; range_end:; response_count:0; response_revision:1195; }","duration":"1.303094116s","start":"2026-03-30T16:50:13.949594Z","end":"2026-03-30T16:50:15.252689Z","steps":["trace[1103635261] 'agreement among raft nodes before linearized reading' (duration: 1.303047056s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.252714Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:13.949557Z","time spent":"1.303153108s","remote":"127.0.0.1:56010","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/resourceclaims\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.252860Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.073187529s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumeclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:15.252907Z","caller":"traceutil/trace.go:172","msg":"trace[1168008724] range","detail":"{range_begin:/registry/persistentvolumeclaims; range_end:; response_count:0; response_revision:1195; }","duration":"3.07323332s","start":"2026-03-30T16:50:12.179662Z","end":"2026-03-30T16:50:15.252895Z","steps":["trace[1168008724] 'agreement among raft nodes before linearized reading' (duration: 3.073160299s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.252934Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.803848506s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:15.252937Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:12.179643Z","time spent":"3.07328545s","remote":"127.0.0.1:55312","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":28,"request content":"key:\"/registry/persistentvolumeclaims\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.252942Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.58482167s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T16:50:15.253027Z","caller":"traceutil/trace.go:172","msg":"trace[360391716] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1195; }","duration":"1.584916931s","start":"2026-03-30T16:50:13.668093Z","end":"2026-03-30T16:50:15.253010Z","steps":["trace[360391716] 'agreement among raft nodes before linearized reading' (duration: 1.584759889s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.253010Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.104667422s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingress\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:15.252864Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.102501461s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/storageclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:15.253108Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.978464898s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T16:50:15.253171Z","caller":"traceutil/trace.go:172","msg":"trace[1859550417] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1195; }","duration":"7.978505648s","start":"2026-03-30T16:50:07.274632Z","end":"2026-03-30T16:50:15.253138Z","steps":["trace[1859550417] 'agreement among raft nodes before linearized reading' (duration: 7.978451347s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.253167Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.055367338s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:15.253208Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:07.274605Z","time spent":"7.978594859s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:15.253253Z","caller":"traceutil/trace.go:172","msg":"trace[216259923] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions; range_end:; response_count:0; response_revision:1195; }","duration":"8.055417019s","start":"2026-03-30T16:50:07.197787Z","end":"2026-03-30T16:50:15.253204Z","steps":["trace[216259923] 'agreement among raft nodes before linearized reading' (duration: 8.055344198s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.253299Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:07.197762Z","time spent":"8.05552274s","remote":"127.0.0.1:55114","response type":"/etcdserverpb.KV/Range","request count":0,"request size":60,"response count":0,"response size":28,"request content":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:15.252961Z","caller":"traceutil/trace.go:172","msg":"trace[294465877] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1195; }","duration":"4.803875116s","start":"2026-03-30T16:50:10.449077Z","end":"2026-03-30T16:50:15.252953Z","steps":["trace[294465877] 'agreement among raft nodes before linearized reading' (duration: 4.803806935s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.252486Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"888.512421ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/replicasets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:15.252545Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"965.769108ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:15.255762Z","caller":"traceutil/trace.go:172","msg":"trace[1398694540] range","detail":"{range_begin:/registry/horizontalpodautoscalers; range_end:; response_count:0; response_revision:1195; }","duration":"968.976162ms","start":"2026-03-30T16:50:14.286767Z","end":"2026-03-30T16:50:15.255743Z","steps":["trace[1398694540] 'agreement among raft nodes before linearized reading' (duration: 965.755878ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.255844Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:14.286739Z","time spent":"969.085305ms","remote":"127.0.0.1:55458","response type":"/etcdserverpb.KV/Range","request count":0,"request size":38,"response count":0,"response size":28,"request content":"key:\"/registry/horizontalpodautoscalers\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.253041Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.993728585s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:50:15.256233Z","caller":"traceutil/trace.go:172","msg":"trace[1147112919] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1195; }","duration":"4.99691421s","start":"2026-03-30T16:50:10.259304Z","end":"2026-03-30T16:50:15.256218Z","steps":["trace[1147112919] 'agreement among raft nodes before linearized reading' (duration: 4.993700715s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.256278Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:10.259270Z","time spent":"4.99699429s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":465,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.256409Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004282,"received-request-id":10218277501228004281} {"level":"info","ts":"2026-03-30T16:50:15.253085Z","caller":"traceutil/trace.go:172","msg":"trace[453857410] range","detail":"{range_begin:/registry/ingress; range_end:; response_count:0; response_revision:1195; }","duration":"2.104747723s","start":"2026-03-30T16:50:13.148320Z","end":"2026-03-30T16:50:15.253068Z","steps":["trace[453857410] 'agreement among raft nodes before linearized reading' (duration: 2.104628212s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.256594Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:13.148292Z","time spent":"2.108290772s","remote":"127.0.0.1:55560","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":28,"request content":"key:\"/registry/ingress\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.253069Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:13.668066Z","time spent":"1.584993901s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:15.253108Z","caller":"traceutil/trace.go:172","msg":"trace[967426365] range","detail":"{range_begin:/registry/storageclasses; range_end:; response_count:0; response_revision:1195; }","duration":"8.102748844s","start":"2026-03-30T16:50:07.150348Z","end":"2026-03-30T16:50:15.253097Z","steps":["trace[967426365] 'agreement among raft nodes before linearized reading' (duration: 8.102488121s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.256817Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:07.150312Z","time spent":"8.106494914s","remote":"127.0.0.1:55706","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/storageclasses\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.253339Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.42050502s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/kube-system/cilium-config\" limit:1 ","response":"range_response_count:1 size:6534"} {"level":"info","ts":"2026-03-30T16:50:15.256874Z","caller":"traceutil/trace.go:172","msg":"trace[1405106557] range","detail":"{range_begin:/registry/configmaps/kube-system/cilium-config; range_end:; response_count:1; response_revision:1195; }","duration":"8.424038027s","start":"2026-03-30T16:50:06.832825Z","end":"2026-03-30T16:50:15.256863Z","steps":["trace[1405106557] 'agreement among raft nodes before linearized reading' (duration: 8.420375158s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.256909Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:06.832816Z","time spent":"8.424083388s","remote":"127.0.0.1:55210","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":1,"response size":6557,"request content":"key:\"/registry/configmaps/kube-system/cilium-config\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.252148Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.100664466s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:15.257080Z","caller":"traceutil/trace.go:172","msg":"trace[1170352344] range","detail":"{range_begin:/registry/pods; range_end:; response_count:0; response_revision:1195; }","duration":"1.105617328s","start":"2026-03-30T16:50:14.151455Z","end":"2026-03-30T16:50:15.257072Z","steps":["trace[1170352344] 'agreement among raft nodes before linearized reading' (duration: 1.100634155s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.257127Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:14.151418Z","time spent":"1.105679409s","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/pods\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:15.253679Z","caller":"traceutil/trace.go:172","msg":"trace[1471408500] range","detail":"{range_begin:/registry/replicasets; range_end:; response_count:0; response_revision:1195; }","duration":"889.692214ms","start":"2026-03-30T16:50:14.363963Z","end":"2026-03-30T16:50:15.253656Z","steps":["trace[1471408500] 'agreement among raft nodes before linearized reading' (duration: 888.493321ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:15.257298Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:14.363944Z","time spent":"893.345974ms","remote":"127.0.0.1:55874","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/replicasets\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:15.264093Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48654","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:50:16.678783Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004289,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:50:16.912807Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"a8d4e8c653fa8dce is starting a new election at term 4"} {"level":"info","ts":"2026-03-30T16:50:16.912886Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"a8d4e8c653fa8dce became pre-candidate at term 4"} {"level":"info","ts":"2026-03-30T16:50:16.912909Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"a8d4e8c653fa8dce [logterm: 4, index: 1429] sent MsgPreVote request to 3cf904ec39d3e333 at term 4"} {"level":"info","ts":"2026-03-30T16:50:16.912921Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"a8d4e8c653fa8dce [logterm: 4, index: 1429] sent MsgPreVote request to c3874f4929b5956e at term 4"} {"level":"info","ts":"2026-03-30T16:50:16.912932Z","logger":"raft","caller":"v3@v3.6.0/node.go:376","msg":"raft.node: a8d4e8c653fa8dce lost leader 3cf904ec39d3e333 at term 4"} {"level":"info","ts":"2026-03-30T16:50:16.913033Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"a8d4e8c653fa8dce received MsgPreVoteResp from a8d4e8c653fa8dce at term 4"} {"level":"info","ts":"2026-03-30T16:50:16.913050Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"a8d4e8c653fa8dce has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T16:50:16.914289Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"a8d4e8c653fa8dce received MsgPreVoteResp from c3874f4929b5956e at term 4"} {"level":"info","ts":"2026-03-30T16:50:16.914333Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"a8d4e8c653fa8dce has received 2 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T16:50:16.914374Z","logger":"raft","caller":"v3@v3.6.0/raft.go:912","msg":"a8d4e8c653fa8dce became candidate at term 5"} {"level":"info","ts":"2026-03-30T16:50:16.914413Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"a8d4e8c653fa8dce [logterm: 4, index: 1429] sent MsgVote request to 3cf904ec39d3e333 at term 5"} {"level":"info","ts":"2026-03-30T16:50:16.914436Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"a8d4e8c653fa8dce [logterm: 4, index: 1429] sent MsgVote request to c3874f4929b5956e at term 5"} {"level":"info","ts":"2026-03-30T16:50:17.084398Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1251","msg":"a8d4e8c653fa8dce [logterm: 4, index: 1429, vote: a8d4e8c653fa8dce] rejected MsgPreVote from c3874f4929b5956e [logterm: 4, index: 1429] at term 5"} {"level":"info","ts":"2026-03-30T16:50:17.096408Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"a8d4e8c653fa8dce received MsgVoteResp from a8d4e8c653fa8dce at term 5"} {"level":"info","ts":"2026-03-30T16:50:17.096494Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"a8d4e8c653fa8dce has received 1 MsgVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-30T16:50:17.179747Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004289,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:50:17.257062Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"a8d4e8c653fa8dce received MsgVoteResp from c3874f4929b5956e at term 5"} {"level":"info","ts":"2026-03-30T16:50:17.257120Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"a8d4e8c653fa8dce has received 2 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T16:50:17.257148Z","logger":"raft","caller":"v3@v3.6.0/raft.go:970","msg":"a8d4e8c653fa8dce became leader at term 5"} {"level":"info","ts":"2026-03-30T16:50:17.257164Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: a8d4e8c653fa8dce elected leader a8d4e8c653fa8dce at term 5"} {"level":"warn","ts":"2026-03-30T16:50:17.257421Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.079091917s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-30T16:50:17.257549Z","caller":"traceutil/trace.go:172","msg":"trace[446796126] range","detail":"{range_begin:; range_end:; }","duration":"1.079243289s","start":"2026-03-30T16:50:16.178288Z","end":"2026-03-30T16:50:17.257531Z","steps":["trace[446796126] 'agreement among raft nodes before linearized reading' (duration: 1.079089747s)"],"step_count":1} {"level":"error","ts":"2026-03-30T16:50:17.257643Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: leader changed\n[+]non_learner ok\n[+]data_corruption ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"info","ts":"2026-03-30T16:50:17.295081Z","caller":"etcdserver/v3_server.go:903","msg":"first commit in current term: resending ReadIndex request"} {"level":"info","ts":"2026-03-30T16:50:17.295033Z","caller":"traceutil/trace.go:172","msg":"trace[1001184507] transaction","detail":"{read_only:false; response_revision:1209; number_of_response:1; }","duration":"1.884388358s","start":"2026-03-30T16:50:15.410621Z","end":"2026-03-30T16:50:17.295009Z","steps":["trace[1001184507] 'process raft request' (duration: 1.884124345s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:17.295698Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:15.410608Z","time spent":"1.884716881s","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7801,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:17.295837Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"961.731349ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:17.295875Z","caller":"traceutil/trace.go:172","msg":"trace[265490062] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1209; }","duration":"961.850061ms","start":"2026-03-30T16:50:16.334014Z","end":"2026-03-30T16:50:17.295864Z","steps":["trace[265490062] 'agreement among raft nodes before linearized reading' (duration: 961.701009ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:17.295905Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:16.333991Z","time spent":"961.908471ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:17.296088Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.022290327s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumnodeconfigs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:17.296118Z","caller":"traceutil/trace.go:172","msg":"trace[1635857558] range","detail":"{range_begin:/registry/cilium.io/ciliumnodeconfigs; range_end:; response_count:0; response_revision:1209; }","duration":"1.022321697s","start":"2026-03-30T16:50:16.273790Z","end":"2026-03-30T16:50:17.296112Z","steps":["trace[1635857558] 'agreement among raft nodes before linearized reading' (duration: 1.022270707s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:17.296135Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:16.273766Z","time spent":"1.022364258s","remote":"127.0.0.1:48662","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":0,"response size":28,"request content":"key:\"/registry/cilium.io/ciliumnodeconfigs\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:17.885552Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"237.70125ms"} {"level":"warn","ts":"2026-03-30T16:50:17.885648Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"237.803582ms"} {"level":"warn","ts":"2026-03-30T16:50:18.194803Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004296,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:18.547908Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"369.405345ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:18.547965Z","caller":"traceutil/trace.go:172","msg":"trace[324853551] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1210; }","duration":"369.479296ms","start":"2026-03-30T16:50:18.178475Z","end":"2026-03-30T16:50:18.547954Z","steps":["trace[324853551] 'range keys from in-memory index tree' (duration: 369.340755ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:18.695679Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004296,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:18.790137Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"395.16279ms"} {"level":"warn","ts":"2026-03-30T16:50:18.790186Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"395.215751ms"} {"level":"info","ts":"2026-03-30T16:50:18.790167Z","caller":"traceutil/trace.go:172","msg":"trace[141105879] linearizableReadLoop","detail":"{readStateIndex:1430; appliedIndex:1431; }","duration":"1.096204645s","start":"2026-03-30T16:50:17.693939Z","end":"2026-03-30T16:50:18.790144Z","steps":["trace[141105879] 'read index received' (duration: 1.096196335s)","trace[141105879] 'applied index is now lower than readState.Index' (duration: 6.83µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:18.790319Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.096359577s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:18.790360Z","caller":"traceutil/trace.go:172","msg":"trace[1772116104] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1210; }","duration":"1.096413547s","start":"2026-03-30T16:50:17.693934Z","end":"2026-03-30T16:50:18.790348Z","steps":["trace[1772116104] 'agreement among raft nodes before linearized reading' (duration: 1.096319976s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:18.790408Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:17.693910Z","time spent":"1.096487688s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:50:18.791253Z","caller":"traceutil/trace.go:172","msg":"trace[1062713839] transaction","detail":"{read_only:false; response_revision:1211; number_of_response:1; }","duration":"1.475907874s","start":"2026-03-30T16:50:17.315326Z","end":"2026-03-30T16:50:18.791234Z","steps":["trace[1062713839] 'process raft request' (duration: 1.475094636s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:18.791364Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:17.315300Z","time spent":"1.476001606s","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18609,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:19.291043Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004298,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:19.337719Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"385.2438ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T16:50:19.750088Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:12.749199Z","time spent":"7.000881983s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-30T16:50:19.791630Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004298,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:20.111576Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"182.681579ms"} {"level":"info","ts":"2026-03-30T16:50:20.111626Z","caller":"traceutil/trace.go:172","msg":"trace[669907463] linearizableReadLoop","detail":"{readStateIndex:1432; appliedIndex:1433; }","duration":"1.321389956s","start":"2026-03-30T16:50:18.790213Z","end":"2026-03-30T16:50:20.111603Z","steps":["trace[669907463] 'read index received' (duration: 1.321378055s)","trace[669907463] 'applied index is now lower than readState.Index' (duration: 10.271µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:20.111687Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"182.79966ms"} {"level":"warn","ts":"2026-03-30T16:50:20.111905Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.778332162s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:20.111982Z","caller":"traceutil/trace.go:172","msg":"trace[994798993] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1212; }","duration":"1.778560164s","start":"2026-03-30T16:50:18.333404Z","end":"2026-03-30T16:50:20.111964Z","steps":["trace[994798993] 'agreement among raft nodes before linearized reading' (duration: 1.778229081s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:20.111977Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.840589897s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-30T16:50:20.112047Z","caller":"traceutil/trace.go:172","msg":"trace[62357376] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1212; }","duration":"1.840677138s","start":"2026-03-30T16:50:18.271355Z","end":"2026-03-30T16:50:20.112032Z","steps":["trace[62357376] 'agreement among raft nodes before linearized reading' (duration: 1.840388175s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:20.112046Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:18.333369Z","time spent":"1.778663686s","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:20.112110Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:18.271305Z","time spent":"1.840771808s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:20.111908Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.563889106s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:20.112172Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.011811838s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T16:50:20.112176Z","caller":"traceutil/trace.go:172","msg":"trace[1483804267] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1212; }","duration":"1.564168859s","start":"2026-03-30T16:50:18.547990Z","end":"2026-03-30T16:50:20.112159Z","steps":["trace[1483804267] 'agreement among raft nodes before linearized reading' (duration: 1.563845306s)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:20.112221Z","caller":"traceutil/trace.go:172","msg":"trace[1710772413] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1212; }","duration":"2.011864519s","start":"2026-03-30T16:50:18.100343Z","end":"2026-03-30T16:50:20.112208Z","steps":["trace[1710772413] 'agreement among raft nodes before linearized reading' (duration: 2.011435024s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:20.112272Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:18.100317Z","time spent":"2.011942919s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:20.112488Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.403573353s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:50:20.112522Z","caller":"traceutil/trace.go:172","msg":"trace[1161553893] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1212; }","duration":"2.403806496s","start":"2026-03-30T16:50:17.708710Z","end":"2026-03-30T16:50:20.112516Z","steps":["trace[1161553893] 'agreement among raft nodes before linearized reading' (duration: 2.403095469s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:20.112542Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:17.708692Z","time spent":"2.403844056s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":465,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:20.539399Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"227.490425ms"} {"level":"warn","ts":"2026-03-30T16:50:20.539418Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004300,"received-request-id":10218277501228004298} {"level":"warn","ts":"2026-03-30T16:50:20.539476Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"227.576266ms"} {"level":"info","ts":"2026-03-30T16:50:20.540994Z","caller":"traceutil/trace.go:172","msg":"trace[977931467] linearizableReadLoop","detail":"{readStateIndex:1435; appliedIndex:1435; }","duration":"429.267653ms","start":"2026-03-30T16:50:20.111710Z","end":"2026-03-30T16:50:20.540978Z","steps":["trace[977931467] 'read index received' (duration: 429.257273ms)","trace[977931467] 'applied index is now lower than readState.Index' (duration: 9.81µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:20.834631Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:13.833696Z","time spent":"7.000927801s","remote":"127.0.0.1:55086","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-30T16:50:20.953800Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.361772994s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T16:50:20.953875Z","caller":"traceutil/trace.go:172","msg":"trace[762152927] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1214; }","duration":"1.361852934s","start":"2026-03-30T16:50:19.592004Z","end":"2026-03-30T16:50:20.953857Z","steps":["trace[762152927] 'agreement among raft nodes before linearized reading' (duration: 949.110594ms)","trace[762152927] 'range keys from in-memory index tree' (duration: 412.536759ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:20.953926Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:19.591981Z","time spent":"1.361936276s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:20.954189Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"414.395568ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:20.954268Z","caller":"traceutil/trace.go:172","msg":"trace[1569029630] linearizableReadLoop","detail":"{readStateIndex:1436; appliedIndex:1435; }","duration":"413.220076ms","start":"2026-03-30T16:50:20.541034Z","end":"2026-03-30T16:50:20.954254Z","steps":["trace[1569029630] 'read index received' (duration: 819.968µs)","trace[1569029630] 'applied index is now lower than readState.Index' (duration: 412.396208ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:20.954331Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"840.248346ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:20.954359Z","caller":"traceutil/trace.go:172","msg":"trace[865920505] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1215; }","duration":"840.277066ms","start":"2026-03-30T16:50:20.114072Z","end":"2026-03-30T16:50:20.954349Z","steps":["trace[865920505] 'agreement among raft nodes before linearized reading' (duration: 840.225756ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:20.954426Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"691.464796ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:20.954520Z","caller":"traceutil/trace.go:172","msg":"trace[549658768] range","detail":"{range_begin:/registry/namespaces; range_end:; response_count:0; response_revision:1215; }","duration":"691.569577ms","start":"2026-03-30T16:50:20.262928Z","end":"2026-03-30T16:50:20.954498Z","steps":["trace[549658768] 'agreement among raft nodes before linearized reading' (duration: 691.410025ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:20.954638Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:20.262900Z","time spent":"691.721318ms","remote":"127.0.0.1:55234","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/namespaces\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:20.954734Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"698.569ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/replicasets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:20.954705Z","caller":"traceutil/trace.go:172","msg":"trace[1543359339] transaction","detail":"{read_only:false; response_revision:1215; number_of_response:1; }","duration":"1.200209457s","start":"2026-03-30T16:50:19.754450Z","end":"2026-03-30T16:50:20.954659Z","steps":["trace[1543359339] 'process raft request' (duration: 785.275864ms)","trace[1543359339] 'compare' (duration: 413.914702ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:20.954776Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"835.604278ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:20.954787Z","caller":"traceutil/trace.go:172","msg":"trace[1334897379] range","detail":"{range_begin:/registry/replicasets; range_end:; response_count:0; response_revision:1215; }","duration":"698.623051ms","start":"2026-03-30T16:50:20.256151Z","end":"2026-03-30T16:50:20.954774Z","steps":["trace[1334897379] 'agreement among raft nodes before linearized reading' (duration: 698.54188ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:20.954816Z","caller":"traceutil/trace.go:172","msg":"trace[175295435] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1215; }","duration":"835.646128ms","start":"2026-03-30T16:50:20.119159Z","end":"2026-03-30T16:50:20.954805Z","steps":["trace[175295435] 'agreement among raft nodes before linearized reading' (duration: 835.576957ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:20.954825Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:20.256129Z","time spent":"698.686822ms","remote":"127.0.0.1:55874","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/replicasets\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:20.954844Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:20.119147Z","time spent":"835.690608ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:20.954940Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:19.754424Z","time spent":"1.200387079s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:21.257055Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"283.569171ms"} {"level":"warn","ts":"2026-03-30T16:50:21.257120Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"283.640862ms"} {"level":"warn","ts":"2026-03-30T16:50:21.462079Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004304,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:21.962853Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004304,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:22.089953Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"832.771397ms","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:40"} {"level":"warn","ts":"2026-03-30T16:50:22.251421Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"794.362689ms"} {"level":"warn","ts":"2026-03-30T16:50:22.251492Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"794.372049ms"} {"level":"info","ts":"2026-03-30T16:50:22.251594Z","caller":"traceutil/trace.go:172","msg":"trace[1160677773] linearizableReadLoop","detail":"{readStateIndex:1436; appliedIndex:1437; }","duration":"1.295141853s","start":"2026-03-30T16:50:20.956412Z","end":"2026-03-30T16:50:22.251554Z","steps":["trace[1160677773] 'read index received' (duration: 1.295122292s)","trace[1160677773] 'applied index is now lower than readState.Index' (duration: 16.53µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:22.684936Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:15.684293Z","time spent":"7.000632116s","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-30T16:50:22.688970Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.732515791s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:22.689036Z","caller":"traceutil/trace.go:172","msg":"trace[1248879260] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1215; }","duration":"1.732612042s","start":"2026-03-30T16:50:20.956404Z","end":"2026-03-30T16:50:22.689016Z","steps":["trace[1248879260] 'agreement among raft nodes before linearized reading' (duration: 1.295397835s)","trace[1248879260] 'range keys from in-memory index tree' (duration: 437.086086ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:22.689135Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"437.52487ms","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:40"} {"level":"warn","ts":"2026-03-30T16:50:22.689276Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:20.859604Z","time spent":"1.82966859s","remote":"127.0.0.1:55086","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-30T16:50:22.784201Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004305,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:22.848440Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"35.572476ms"} {"level":"warn","ts":"2026-03-30T16:50:22.848543Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"35.684277ms"} {"level":"warn","ts":"2026-03-30T16:50:22.956646Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.99989904s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-30T16:50:22.956827Z","caller":"traceutil/trace.go:172","msg":"trace[801053623] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000091552s","start":"2026-03-30T16:50:20.956711Z","end":"2026-03-30T16:50:22.956802Z","steps":["trace[801053623] 'agreement among raft nodes before linearized reading' (duration: 1.99989391s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:22.956952Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:20.956689Z","time spent":"2.000251085s","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 16:50:22 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:23.123532Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"75.088368ms"} {"level":"warn","ts":"2026-03-30T16:50:23.123598Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"75.054798ms"} {"level":"warn","ts":"2026-03-30T16:50:23.284316Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004305,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:50:23.543785Z","caller":"traceutil/trace.go:172","msg":"trace[242984917] linearizableReadLoop","detail":"{readStateIndex:1438; appliedIndex:1440; }","duration":"1.292045988s","start":"2026-03-30T16:50:22.251693Z","end":"2026-03-30T16:50:23.543739Z","steps":["trace[242984917] 'read index received' (duration: 1.292038828s)","trace[242984917] 'applied index is now lower than readState.Index' (duration: 6.2µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:23.544150Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.336890378s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:50:23.544187Z","caller":"traceutil/trace.go:172","msg":"trace[16257564] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1217; }","duration":"1.336941518s","start":"2026-03-30T16:50:22.207234Z","end":"2026-03-30T16:50:23.544176Z","steps":["trace[16257564] 'agreement among raft nodes before linearized reading' (duration: 1.336753556s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:23.544214Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:22.207199Z","time spent":"1.337008829s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":465,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:23.544206Z","caller":"traceutil/trace.go:172","msg":"trace[1876439972] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1218; }","duration":"854.007932ms","start":"2026-03-30T16:50:22.690178Z","end":"2026-03-30T16:50:23.544186Z","steps":["trace[1876439972] 'process raft request' (duration: 853.964972ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:23.544281Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.991428336s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:18624"} {"level":"info","ts":"2026-03-30T16:50:23.544268Z","caller":"traceutil/trace.go:172","msg":"trace[740089641] transaction","detail":"{read_only:false; response_revision:1218; number_of_response:1; }","duration":"854.593478ms","start":"2026-03-30T16:50:22.689657Z","end":"2026-03-30T16:50:23.544250Z","steps":["trace[740089641] 'process raft request' (duration: 854.356356ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:23.544358Z","caller":"traceutil/trace.go:172","msg":"trace[1229469061] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:1218; }","duration":"1.991513547s","start":"2026-03-30T16:50:21.552827Z","end":"2026-03-30T16:50:23.544341Z","steps":["trace[1229469061] 'agreement among raft nodes before linearized reading' (duration: 1.991289535s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:23.544430Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:21.552803Z","time spent":"1.991614878s","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18647,"request content":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:23.544390Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:22.690163Z","time spent":"854.151854ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":41,"response count":0,"response size":37,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:23.544237Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.1253743s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/roles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:23.544529Z","caller":"traceutil/trace.go:172","msg":"trace[433877096] range","detail":"{range_begin:/registry/roles; range_end:; response_count:0; response_revision:1218; }","duration":"2.125685103s","start":"2026-03-30T16:50:21.418831Z","end":"2026-03-30T16:50:23.544516Z","steps":["trace[433877096] 'agreement among raft nodes before linearized reading' (duration: 2.125274159s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:23.544551Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:22.689626Z","time spent":"854.776781ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":739,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:23.544566Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:21.418808Z","time spent":"2.125747764s","remote":"127.0.0.1:55636","response type":"/etcdserverpb.KV/Range","request count":0,"request size":19,"response count":0,"response size":28,"request content":"key:\"/registry/roles\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:23.726827Z","caller":"traceutil/trace.go:172","msg":"trace[82376922] linearizableReadLoop","detail":"{readStateIndex:1442; appliedIndex:1442; }","duration":"182.943803ms","start":"2026-03-30T16:50:23.543866Z","end":"2026-03-30T16:50:23.726810Z","steps":["trace[82376922] 'read index received' (duration: 182.939393ms)","trace[82376922] 'applied index is now lower than readState.Index' (duration: 3.8µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:23.796186Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"830.727685ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:23.796252Z","caller":"traceutil/trace.go:172","msg":"trace[1790044924] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1218; }","duration":"830.797496ms","start":"2026-03-30T16:50:22.965441Z","end":"2026-03-30T16:50:23.796238Z","steps":["trace[1790044924] 'agreement among raft nodes before linearized reading' (duration: 761.41314ms)","trace[1790044924] 'range keys from in-memory index tree' (duration: 69.270395ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:23.796233Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.105778104s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:23.796280Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:22.965420Z","time spent":"830.853036ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:50:23.796328Z","caller":"traceutil/trace.go:172","msg":"trace[730958293] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1218; }","duration":"1.105875445s","start":"2026-03-30T16:50:22.690420Z","end":"2026-03-30T16:50:23.796296Z","steps":["trace[730958293] 'agreement among raft nodes before linearized reading' (duration: 1.03647557s)","trace[730958293] 'range keys from in-memory index tree' (duration: 69.274814ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:23.796472Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.484089065s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-30T16:50:23.796626Z","caller":"traceutil/trace.go:172","msg":"trace[774815014] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1218; }","duration":"1.484391658s","start":"2026-03-30T16:50:22.312216Z","end":"2026-03-30T16:50:23.796608Z","steps":["trace[774815014] 'agreement among raft nodes before linearized reading' (duration: 1.41468519s)","trace[774815014] 'range keys from in-memory index tree' (duration: 69.255614ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:23.796666Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:22.312196Z","time spent":"1.484458759s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:23.869675Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:23.546066Z","time spent":"323.604873ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-30T16:50:24.227746Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004313,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:50:24.251246Z","caller":"traceutil/trace.go:172","msg":"trace[1783261640] linearizableReadLoop","detail":"{readStateIndex:1443; appliedIndex:1444; }","duration":"524.33012ms","start":"2026-03-30T16:50:23.726889Z","end":"2026-03-30T16:50:24.251219Z","steps":["trace[1783261640] 'read index received' (duration: 524.29679ms)","trace[1783261640] 'applied index is now lower than readState.Index' (duration: 31.95µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:24.251364Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"181.944961ms"} {"level":"warn","ts":"2026-03-30T16:50:24.251386Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"181.977411ms"} {"level":"warn","ts":"2026-03-30T16:50:24.251524Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"532.342223ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:24.251619Z","caller":"traceutil/trace.go:172","msg":"trace[1623762736] range","detail":"{range_begin:/registry/services/endpoints; range_end:; response_count:0; response_revision:1219; }","duration":"532.439214ms","start":"2026-03-30T16:50:23.719147Z","end":"2026-03-30T16:50:24.251586Z","steps":["trace[1623762736] 'agreement among raft nodes before linearized reading' (duration: 532.275322ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.251669Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:23.719128Z","time spent":"532.532395ms","remote":"127.0.0.1:55346","response type":"/etcdserverpb.KV/Range","request count":0,"request size":32,"response count":0,"response size":28,"request content":"key:\"/registry/services/endpoints\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:24.252004Z","caller":"traceutil/trace.go:172","msg":"trace[80127507] transaction","detail":"{read_only:false; response_revision:1220; number_of_response:1; }","duration":"704.16956ms","start":"2026-03-30T16:50:23.547822Z","end":"2026-03-30T16:50:24.251992Z","steps":["trace[80127507] 'process raft request' (duration: 703.912278ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:24.251997Z","caller":"traceutil/trace.go:172","msg":"trace[1024669162] transaction","detail":"{read_only:false; response_revision:1221; number_of_response:1; }","duration":"690.086763ms","start":"2026-03-30T16:50:23.561893Z","end":"2026-03-30T16:50:24.251980Z","steps":["trace[1024669162] 'process raft request' (duration: 690.014702ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.252069Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:23.547800Z","time spent":"704.233441ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":709,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:24.252118Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:23.561867Z","time spent":"690.184184ms","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18875,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:24.446883Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004315,"received-request-id":10218277501228004313} {"level":"warn","ts":"2026-03-30T16:50:24.620045Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"300.804872ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:24.620317Z","caller":"traceutil/trace.go:172","msg":"trace[2134155579] transaction","detail":"{read_only:false; response_revision:1226; number_of_response:1; }","duration":"364.275895ms","start":"2026-03-30T16:50:24.256029Z","end":"2026-03-30T16:50:24.620305Z","steps":["trace[2134155579] 'process raft request' (duration: 364.201784ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.620369Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:24.256010Z","time spent":"364.342385ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":708,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T16:50:24.620651Z","caller":"traceutil/trace.go:172","msg":"trace[187532812] linearizableReadLoop","detail":"{readStateIndex:1448; appliedIndex:1446; }","duration":"369.329787ms","start":"2026-03-30T16:50:24.251314Z","end":"2026-03-30T16:50:24.620643Z","steps":["trace[187532812] 'read index received' (duration: 223.327622ms)","trace[187532812] 'applied index is now lower than readState.Index' (duration: 146.001335ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:24.620705Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"823.141569ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:24.620722Z","caller":"traceutil/trace.go:172","msg":"trace[768732785] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1227; }","duration":"823.163449ms","start":"2026-03-30T16:50:23.797554Z","end":"2026-03-30T16:50:24.620717Z","steps":["trace[768732785] 'agreement among raft nodes before linearized reading' (duration: 823.121399ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.620737Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:23.797533Z","time spent":"823.200099ms","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:50:24.620840Z","caller":"traceutil/trace.go:172","msg":"trace[1087832129] transaction","detail":"{read_only:false; response_revision:1225; number_of_response:1; }","duration":"749.473099ms","start":"2026-03-30T16:50:23.871360Z","end":"2026-03-30T16:50:24.620833Z","steps":["trace[1087832129] 'process raft request' (duration: 748.851133ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.620894Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:23.871334Z","time spent":"749.53384ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:24.621055Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"649.261852ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T16:50:24.621079Z","caller":"traceutil/trace.go:172","msg":"trace[951218122] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1227; }","duration":"649.286492ms","start":"2026-03-30T16:50:23.971786Z","end":"2026-03-30T16:50:24.621073Z","steps":["trace[951218122] 'agreement among raft nodes before linearized reading' (duration: 649.206871ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.621093Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:23.971752Z","time spent":"649.335883ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:24.621158Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"744.561358ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csistoragecapacities\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:24.621221Z","caller":"traceutil/trace.go:172","msg":"trace[1318359554] range","detail":"{range_begin:/registry/csistoragecapacities; range_end:; response_count:0; response_revision:1227; }","duration":"744.620929ms","start":"2026-03-30T16:50:23.876579Z","end":"2026-03-30T16:50:24.621199Z","steps":["trace[1318359554] 'agreement among raft nodes before linearized reading' (duration: 744.533118ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.621376Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:23.876556Z","time spent":"744.812041ms","remote":"127.0.0.1:55788","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":28,"request content":"key:\"/registry/csistoragecapacities\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:24.621458Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"823.402643ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:24.621853Z","caller":"traceutil/trace.go:172","msg":"trace[395556700] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1227; }","duration":"823.940438ms","start":"2026-03-30T16:50:23.797890Z","end":"2026-03-30T16:50:24.621830Z","steps":["trace[395556700] 'agreement among raft nodes before linearized reading' (duration: 823.331002ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:24.868565Z","caller":"traceutil/trace.go:172","msg":"trace[1198911699] linearizableReadLoop","detail":"{readStateIndex:1452; appliedIndex:1452; }","duration":"245.983388ms","start":"2026-03-30T16:50:24.622548Z","end":"2026-03-30T16:50:24.868531Z","steps":["trace[1198911699] 'read index received' (duration: 245.974248ms)","trace[1198911699] 'applied index is now lower than readState.Index' (duration: 7.64µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:24.889716Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"267.143429ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:422"} {"level":"info","ts":"2026-03-30T16:50:24.889778Z","caller":"traceutil/trace.go:172","msg":"trace[116524970] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1227; }","duration":"267.22269ms","start":"2026-03-30T16:50:24.622542Z","end":"2026-03-30T16:50:24.889765Z","steps":["trace[116524970] 'agreement among raft nodes before linearized reading' (duration: 246.304202ms)","trace[116524970] 'range keys from in-memory index tree' (duration: 20.718526ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:24.890303Z","caller":"traceutil/trace.go:172","msg":"trace[1275314152] transaction","detail":"{read_only:false; response_revision:1230; number_of_response:1; }","duration":"265.977498ms","start":"2026-03-30T16:50:24.624306Z","end":"2026-03-30T16:50:24.890283Z","steps":["trace[1275314152] 'process raft request' (duration: 265.748745ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.890605Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"267.618785ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:24.890668Z","caller":"traceutil/trace.go:172","msg":"trace[394159416] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1231; }","duration":"267.688606ms","start":"2026-03-30T16:50:24.622966Z","end":"2026-03-30T16:50:24.890655Z","steps":["trace[394159416] 'agreement among raft nodes before linearized reading' (duration: 267.599975ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:24.890934Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"262.773254ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:24.891249Z","caller":"traceutil/trace.go:172","msg":"trace[1573618967] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1231; }","duration":"263.095997ms","start":"2026-03-30T16:50:24.628142Z","end":"2026-03-30T16:50:24.891238Z","steps":["trace[1573618967] 'agreement among raft nodes before linearized reading' (duration: 262.473341ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:25.392486Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004320,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:25.470224Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"393.563174ms"} {"level":"warn","ts":"2026-03-30T16:50:25.470461Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"393.806136ms"} {"level":"warn","ts":"2026-03-30T16:50:25.471162Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"282.720349ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:25.471266Z","caller":"traceutil/trace.go:172","msg":"trace[1489876248] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1231; }","duration":"282.8367ms","start":"2026-03-30T16:50:25.188415Z","end":"2026-03-30T16:50:25.471251Z","steps":["trace[1489876248] 'range keys from in-memory index tree' (duration: 282.325195ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:25.602739Z","caller":"traceutil/trace.go:172","msg":"trace[392567277] linearizableReadLoop","detail":"{readStateIndex:1456; appliedIndex:1457; }","duration":"711.104269ms","start":"2026-03-30T16:50:24.891611Z","end":"2026-03-30T16:50:25.602715Z","steps":["trace[392567277] 'read index received' (duration: 711.092108ms)","trace[392567277] 'applied index is now lower than readState.Index' (duration: 10.751µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:26.103934Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004322,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:26.360047Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.468418616s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:2 size:263"} {"level":"info","ts":"2026-03-30T16:50:26.360128Z","caller":"traceutil/trace.go:172","msg":"trace[1679667389] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:2; response_revision:1232; }","duration":"1.468499067s","start":"2026-03-30T16:50:24.891607Z","end":"2026-03-30T16:50:26.360106Z","steps":["trace[1679667389] 'agreement among raft nodes before linearized reading' (duration: 711.23779ms)","trace[1679667389] 'range keys from in-memory index tree' (duration: 757.073055ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:26.360166Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:24.891591Z","time spent":"1.468565048s","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":2,"response size":286,"request content":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" "} {"level":"warn","ts":"2026-03-30T16:50:26.577609Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"774.892456ms"} {"level":"warn","ts":"2026-03-30T16:50:26.577710Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"775.016147ms"} {"level":"info","ts":"2026-03-30T16:50:26.578267Z","caller":"traceutil/trace.go:172","msg":"trace[512835216] transaction","detail":"{read_only:false; response_revision:1235; number_of_response:1; }","duration":"1.293601979s","start":"2026-03-30T16:50:25.284647Z","end":"2026-03-30T16:50:26.578249Z","steps":["trace[512835216] 'process raft request' (duration: 1.293313066s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:26.578357Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:25.284616Z","time spent":"1.29368265s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:26.605095Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004322,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:26.892456Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000152086s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T16:50:26.892526Z","caller":"traceutil/trace.go:172","msg":"trace[2139594036] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000234377s","start":"2026-03-30T16:50:24.892278Z","end":"2026-03-30T16:50:26.892513Z","steps":["trace[2139594036] 'agreement among raft nodes before linearized reading' (duration: 2.000149546s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:26.892567Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:24.892266Z","time spent":"2.000295198s","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 16:50:26 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T16:50:27.042307Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"264.56247ms"} {"level":"warn","ts":"2026-03-30T16:50:27.042423Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"264.698521ms"} {"level":"info","ts":"2026-03-30T16:50:27.042351Z","caller":"traceutil/trace.go:172","msg":"trace[1767784722] linearizableReadLoop","detail":"{readStateIndex:1457; appliedIndex:1461; }","duration":"1.439515905s","start":"2026-03-30T16:50:25.602803Z","end":"2026-03-30T16:50:27.042319Z","steps":["trace[1767784722] 'read index received' (duration: 1.439501685s)","trace[1767784722] 'applied index is now lower than readState.Index' (duration: 12.76µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:27.042466Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.571097517s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:27.042852Z","caller":"traceutil/trace.go:172","msg":"trace[1778167935] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1236; }","duration":"1.571493901s","start":"2026-03-30T16:50:25.471338Z","end":"2026-03-30T16:50:27.042832Z","steps":["trace[1778167935] 'agreement among raft nodes before linearized reading' (duration: 1.571060057s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:27.042605Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.147755694s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a1ac83777bb378\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-30T16:50:27.043070Z","caller":"traceutil/trace.go:172","msg":"trace[1278324148] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18a1ac83777bb378; range_end:; response_count:1; response_revision:1236; }","duration":"2.148198989s","start":"2026-03-30T16:50:24.894812Z","end":"2026-03-30T16:50:27.043011Z","steps":["trace[1278324148] 'agreement among raft nodes before linearized reading' (duration: 2.147593763s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:27.043664Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:24.894803Z","time spent":"2.148808325s","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":817,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a1ac83777bb378\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:27.513898Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"271.106934ms"} {"level":"warn","ts":"2026-03-30T16:50:27.513972Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"271.190375ms"} {"level":"info","ts":"2026-03-30T16:50:27.513982Z","caller":"traceutil/trace.go:172","msg":"trace[1455815701] linearizableReadLoop","detail":"{readStateIndex:1461; appliedIndex:1461; }","duration":"471.222776ms","start":"2026-03-30T16:50:27.042731Z","end":"2026-03-30T16:50:27.513954Z","steps":["trace[1455815701] 'read index received' (duration: 471.211965ms)","trace[1455815701] 'applied index is now lower than readState.Index' (duration: 9.491µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:27.514202Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"710.510948ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:50:27.514282Z","caller":"traceutil/trace.go:172","msg":"trace[626210072] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1236; }","duration":"710.606929ms","start":"2026-03-30T16:50:26.803658Z","end":"2026-03-30T16:50:27.514265Z","steps":["trace[626210072] 'agreement among raft nodes before linearized reading' (duration: 710.345906ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:27.514298Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"613.097292ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:27.514328Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:26.803603Z","time spent":"710.713079ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":465,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:27.514334Z","caller":"traceutil/trace.go:172","msg":"trace[966049017] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1236; }","duration":"613.128712ms","start":"2026-03-30T16:50:26.901193Z","end":"2026-03-30T16:50:27.514322Z","steps":["trace[966049017] 'agreement among raft nodes before linearized reading' (duration: 613.071542ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:27.514369Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"810.044986ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:18890"} {"level":"warn","ts":"2026-03-30T16:50:27.514372Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:26.901135Z","time spent":"613.230215ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:27.514221Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.733522842s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:27.514353Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.152028028s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:50:27.514405Z","caller":"traceutil/trace.go:172","msg":"trace[1304928108] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:1236; }","duration":"810.093576ms","start":"2026-03-30T16:50:26.704299Z","end":"2026-03-30T16:50:27.514392Z","steps":["trace[1304928108] 'agreement among raft nodes before linearized reading' (duration: 809.914774ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:27.514445Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:26.704280Z","time spent":"810.158996ms","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18913,"request content":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:27.514442Z","caller":"traceutil/trace.go:172","msg":"trace[683203653] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1236; }","duration":"1.152139369s","start":"2026-03-30T16:50:26.362289Z","end":"2026-03-30T16:50:27.514428Z","steps":["trace[683203653] 'agreement among raft nodes before linearized reading' (duration: 1.151852596s)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:27.514448Z","caller":"traceutil/trace.go:172","msg":"trace[1557585692] range","detail":"{range_begin:/registry/services/endpoints; range_end:; response_count:0; response_revision:1236; }","duration":"1.733760354s","start":"2026-03-30T16:50:25.780655Z","end":"2026-03-30T16:50:27.514416Z","steps":["trace[1557585692] 'agreement among raft nodes before linearized reading' (duration: 1.733471271s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:27.514501Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:26.362274Z","time spent":"1.1522132s","remote":"127.0.0.1:55346","response type":"/etcdserverpb.KV/Range","request count":0,"request size":51,"response count":1,"response size":465,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:27.514521Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:25.780637Z","time spent":"1.733866536s","remote":"127.0.0.1:55346","response type":"/etcdserverpb.KV/Range","request count":0,"request size":32,"response count":0,"response size":28,"request content":"key:\"/registry/services/endpoints\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:27.514443Z","caller":"traceutil/trace.go:172","msg":"trace[1005801843] transaction","detail":"{read_only:false; response_revision:1239; number_of_response:1; }","duration":"468.040082ms","start":"2026-03-30T16:50:27.046388Z","end":"2026-03-30T16:50:27.514428Z","steps":["trace[1005801843] 'process raft request' (duration: 467.998932ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:27.514766Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:27.046359Z","time spent":"468.283606ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:28.014505Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004325,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:28.515649Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004325,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:28.718512Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"805.235895ms"} {"level":"warn","ts":"2026-03-30T16:50:28.718580Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"805.312536ms"} {"level":"info","ts":"2026-03-30T16:50:28.719152Z","caller":"traceutil/trace.go:172","msg":"trace[1210158869] linearizableReadLoop","detail":"{readStateIndex:1473; appliedIndex:1471; }","duration":"1.205067414s","start":"2026-03-30T16:50:27.514067Z","end":"2026-03-30T16:50:28.719134Z","steps":["trace[1210158869] 'read index received' (duration: 1.204903543s)","trace[1210158869] 'applied index is now lower than readState.Index' (duration: 162.861µs)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:28.719233Z","caller":"traceutil/trace.go:172","msg":"trace[1747380645] transaction","detail":"{read_only:false; response_revision:1244; number_of_response:1; }","duration":"1.184273947s","start":"2026-03-30T16:50:27.534950Z","end":"2026-03-30T16:50:28.719224Z","steps":["trace[1747380645] 'process raft request' (duration: 1.184079945s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:28.719353Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:27.534920Z","time spent":"1.184344439s","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":19048,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:28.719363Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.67523115s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:28.719524Z","caller":"traceutil/trace.go:172","msg":"trace[325132764] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1245; }","duration":"1.675395721s","start":"2026-03-30T16:50:27.044113Z","end":"2026-03-30T16:50:28.719509Z","steps":["trace[325132764] 'agreement among raft nodes before linearized reading' (duration: 1.675207379s)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:28.719465Z","caller":"traceutil/trace.go:172","msg":"trace[26103119] transaction","detail":"{read_only:false; response_revision:1243; number_of_response:1; }","duration":"1.199526138s","start":"2026-03-30T16:50:27.519870Z","end":"2026-03-30T16:50:28.719396Z","steps":["trace[26103119] 'process raft request' (duration: 1.199042892s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:28.720238Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:27.519859Z","time spent":"1.199987693s","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":731,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:29.219529Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004328,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:29.314000Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"395.392359ms"} {"level":"warn","ts":"2026-03-30T16:50:29.314096Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"395.4978ms"} {"level":"info","ts":"2026-03-30T16:50:29.468721Z","caller":"traceutil/trace.go:172","msg":"trace[177937855] linearizableReadLoop","detail":"{readStateIndex:1474; appliedIndex:1474; }","duration":"749.504929ms","start":"2026-03-30T16:50:28.719188Z","end":"2026-03-30T16:50:29.468693Z","steps":["trace[177937855] 'read index received' (duration: 749.491059ms)","trace[177937855] 'applied index is now lower than readState.Index' (duration: 12.04µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:29.469165Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"972.940622ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"warn","ts":"2026-03-30T16:50:29.469160Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.110978787s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:29.469207Z","caller":"traceutil/trace.go:172","msg":"trace[75194863] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1245; }","duration":"973.002782ms","start":"2026-03-30T16:50:28.496195Z","end":"2026-03-30T16:50:29.469197Z","steps":["trace[75194863] 'agreement among raft nodes before linearized reading' (duration: 972.647929ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:29.469214Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.452441187s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:434"} {"level":"info","ts":"2026-03-30T16:50:29.469332Z","caller":"traceutil/trace.go:172","msg":"trace[1719198696] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1245; }","duration":"1.452582448s","start":"2026-03-30T16:50:28.016733Z","end":"2026-03-30T16:50:29.469315Z","steps":["trace[1719198696] 'agreement among raft nodes before linearized reading' (duration: 1.452137574s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:29.469382Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.952051977s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:29.469384Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:28.016709Z","time spent":"1.452661789s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":457,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:29.469291Z","caller":"traceutil/trace.go:172","msg":"trace[1636608121] range","detail":"{range_begin:/registry/leases; range_end:; response_count:0; response_revision:1245; }","duration":"1.111062517s","start":"2026-03-30T16:50:28.358151Z","end":"2026-03-30T16:50:29.469214Z","steps":["trace[1636608121] 'agreement among raft nodes before linearized reading' (duration: 1.110748044s)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:29.469416Z","caller":"traceutil/trace.go:172","msg":"trace[1657177517] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1245; }","duration":"1.952075847s","start":"2026-03-30T16:50:27.517325Z","end":"2026-03-30T16:50:29.469401Z","steps":["trace[1657177517] 'agreement among raft nodes before linearized reading' (duration: 1.952036837s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:29.469495Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:28.358128Z","time spent":"1.111351951s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":20,"response count":0,"response size":28,"request content":"key:\"/registry/leases\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:29.469347Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.95139431s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.227\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-30T16:50:29.469676Z","caller":"traceutil/trace.go:172","msg":"trace[833787129] range","detail":"{range_begin:/registry/masterleases/199.204.45.227; range_end:; response_count:1; response_revision:1245; }","duration":"1.951716003s","start":"2026-03-30T16:50:27.517948Z","end":"2026-03-30T16:50:29.469663Z","steps":["trace[833787129] 'agreement among raft nodes before linearized reading' (duration: 1.951343739s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:29.469709Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:27.517937Z","time spent":"1.951761743s","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.227\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:29.469801Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.274275724s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"warn","ts":"2026-03-30T16:50:29.469498Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:27.517309Z","time spent":"1.952174158s","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:50:29.469855Z","caller":"traceutil/trace.go:172","msg":"trace[109033627] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1248; }","duration":"1.274330525s","start":"2026-03-30T16:50:28.195511Z","end":"2026-03-30T16:50:29.469841Z","steps":["trace[109033627] 'agreement among raft nodes before linearized reading' (duration: 1.274037792s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:29.469894Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:28.195490Z","time spent":"1.274392466s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:29.469272Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:28.496174Z","time spent":"973.088284ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:29.968947Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004330,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:30.045848Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"377.278045ms"} {"level":"warn","ts":"2026-03-30T16:50:30.045915Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"377.353426ms"} {"level":"info","ts":"2026-03-30T16:50:30.046044Z","caller":"traceutil/trace.go:172","msg":"trace[1689818676] transaction","detail":"{read_only:false; response_revision:1249; number_of_response:1; }","duration":"1.320985393s","start":"2026-03-30T16:50:28.725043Z","end":"2026-03-30T16:50:30.046028Z","steps":["trace[1689818676] 'process raft request' (duration: 1.320897192s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.046144Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:28.725033Z","time spent":"1.321068705s","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":730,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T16:50:30.316778Z","caller":"traceutil/trace.go:172","msg":"trace[2057301643] linearizableReadLoop","detail":"{readStateIndex:1477; appliedIndex:1480; }","duration":"847.951425ms","start":"2026-03-30T16:50:29.468800Z","end":"2026-03-30T16:50:30.316751Z","steps":["trace[2057301643] 'read index received' (duration: 847.942005ms)","trace[2057301643] 'applied index is now lower than readState.Index' (duration: 8.22µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:30.316938Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"947.339846ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/podtemplates\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:30.316996Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.596329122s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:30.317028Z","caller":"traceutil/trace.go:172","msg":"trace[1220216573] range","detail":"{range_begin:/registry/podtemplates; range_end:; response_count:0; response_revision:1251; }","duration":"947.446737ms","start":"2026-03-30T16:50:29.369563Z","end":"2026-03-30T16:50:30.317009Z","steps":["trace[1220216573] 'agreement among raft nodes before linearized reading' (duration: 947.266146ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.317052Z","caller":"traceutil/trace.go:172","msg":"trace[1457542169] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1251; }","duration":"1.596407433s","start":"2026-03-30T16:50:28.720631Z","end":"2026-03-30T16:50:30.317039Z","steps":["trace[1457542169] 'agreement among raft nodes before linearized reading' (duration: 1.596296992s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.317077Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:29.369542Z","time spent":"947.525819ms","remote":"127.0.0.1:55254","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/podtemplates\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:30.317347Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:29.471715Z","time spent":"845.614191ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-30T16:50:30.317427Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.593658123s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:19063"} {"level":"warn","ts":"2026-03-30T16:50:30.318167Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"824.884864ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:30.318221Z","caller":"traceutil/trace.go:172","msg":"trace[1297138545] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1252; }","duration":"824.939744ms","start":"2026-03-30T16:50:29.493269Z","end":"2026-03-30T16:50:30.318208Z","steps":["trace[1297138545] 'agreement among raft nodes before linearized reading' (duration: 824.855764ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.318265Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:29.493251Z","time spent":"825.004056ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:30.318350Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"287.496223ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:30.318401Z","caller":"traceutil/trace.go:172","msg":"trace[892597437] range","detail":"{range_begin:/registry/configmaps; range_end:; response_count:0; response_revision:1252; }","duration":"287.555953ms","start":"2026-03-30T16:50:30.030828Z","end":"2026-03-30T16:50:30.318384Z","steps":["trace[892597437] 'agreement among raft nodes before linearized reading' (duration: 287.348151ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.317709Z","caller":"traceutil/trace.go:172","msg":"trace[821407922] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:1251; }","duration":"1.593941766s","start":"2026-03-30T16:50:28.723751Z","end":"2026-03-30T16:50:30.317692Z","steps":["trace[821407922] 'agreement among raft nodes before linearized reading' (duration: 1.593078688s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.318487Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"387.644583ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:30.318537Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:28.723738Z","time spent":"1.594778655s","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":19086,"request content":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:30.318560Z","caller":"traceutil/trace.go:172","msg":"trace[982138246] range","detail":"{range_begin:/registry/services/specs; range_end:; response_count:0; response_revision:1252; }","duration":"387.720174ms","start":"2026-03-30T16:50:29.930827Z","end":"2026-03-30T16:50:30.318548Z","steps":["trace[982138246] 'agreement among raft nodes before linearized reading' (duration: 387.563132ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.318592Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"775.34213ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-03-30T16:50:30.318622Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:29.930805Z","time spent":"387.779044ms","remote":"127.0.0.1:55410","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/services/specs\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:30.318638Z","caller":"traceutil/trace.go:172","msg":"trace[1075692158] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1252; }","duration":"775.389321ms","start":"2026-03-30T16:50:29.543236Z","end":"2026-03-30T16:50:30.318625Z","steps":["trace[1075692158] 'agreement among raft nodes before linearized reading' (duration: 775.250349ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.318676Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:29.543221Z","time spent":"775.444181ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:30.317196Z","caller":"traceutil/trace.go:172","msg":"trace[517574496] transaction","detail":"{read_only:false; response_revision:1252; number_of_response:1; }","duration":"826.150278ms","start":"2026-03-30T16:50:29.491024Z","end":"2026-03-30T16:50:30.317174Z","steps":["trace[517574496] 'process raft request' (duration: 825.991966ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.318832Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:29.490999Z","time spent":"827.757164ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":466,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:30.319501Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"269.944292ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a1ac83777bb378\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-30T16:50:30.319857Z","caller":"traceutil/trace.go:172","msg":"trace[815591236] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18a1ac83777bb378; range_end:; response_count:1; response_revision:1252; }","duration":"270.294656ms","start":"2026-03-30T16:50:30.049541Z","end":"2026-03-30T16:50:30.319836Z","steps":["trace[815591236] 'agreement among raft nodes before linearized reading' (duration: 268.761999ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.451224Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"132.03966ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:30.451671Z","caller":"traceutil/trace.go:172","msg":"trace[2115119105] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1252; }","duration":"132.168561ms","start":"2026-03-30T16:50:30.319147Z","end":"2026-03-30T16:50:30.451315Z","steps":["trace[2115119105] 'agreement among raft nodes before linearized reading' (duration: 66.918274ms)","trace[2115119105] 'range keys from in-memory index tree' (duration: 65.057814ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:30.509056Z","caller":"traceutil/trace.go:172","msg":"trace[70470551] linearizableReadLoop","detail":"{readStateIndex:1482; appliedIndex:1482; }","duration":"123.002026ms","start":"2026-03-30T16:50:30.386016Z","end":"2026-03-30T16:50:30.509018Z","steps":["trace[70470551] 'read index received' (duration: 122.992556ms)","trace[70470551] 'applied index is now lower than readState.Index' (duration: 8.09µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:30.509211Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"189.005661ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:30.509310Z","caller":"traceutil/trace.go:172","msg":"trace[450963578] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1252; }","duration":"189.117332ms","start":"2026-03-30T16:50:30.320177Z","end":"2026-03-30T16:50:30.509294Z","steps":["trace[450963578] 'agreement among raft nodes before linearized reading' (duration: 188.95324ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.509678Z","caller":"traceutil/trace.go:172","msg":"trace[1446690845] transaction","detail":"{read_only:false; response_revision:1253; number_of_response:1; }","duration":"190.388295ms","start":"2026-03-30T16:50:30.319266Z","end":"2026-03-30T16:50:30.509654Z","steps":["trace[1446690845] 'process raft request' (duration: 190.152743ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.509809Z","caller":"traceutil/trace.go:172","msg":"trace[1305494218] transaction","detail":"{read_only:false; response_revision:1254; number_of_response:1; }","duration":"186.496096ms","start":"2026-03-30T16:50:30.323290Z","end":"2026-03-30T16:50:30.509786Z","steps":["trace[1305494218] 'process raft request' (duration: 186.288644ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.509875Z","caller":"traceutil/trace.go:172","msg":"trace[790953978] transaction","detail":"{read_only:false; response_revision:1259; number_of_response:1; }","duration":"157.864148ms","start":"2026-03-30T16:50:30.351995Z","end":"2026-03-30T16:50:30.509860Z","steps":["trace[790953978] 'process raft request' (duration: 157.829728ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.510119Z","caller":"traceutil/trace.go:172","msg":"trace[1454283068] transaction","detail":"{read_only:false; response_revision:1256; number_of_response:1; }","duration":"182.614255ms","start":"2026-03-30T16:50:30.327482Z","end":"2026-03-30T16:50:30.510097Z","steps":["trace[1454283068] 'process raft request' (duration: 182.253241ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.510229Z","caller":"traceutil/trace.go:172","msg":"trace[1659358563] transaction","detail":"{read_only:false; response_revision:1258; number_of_response:1; }","duration":"176.647713ms","start":"2026-03-30T16:50:30.333564Z","end":"2026-03-30T16:50:30.510212Z","steps":["trace[1659358563] 'process raft request' (duration: 176.233949ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.510475Z","caller":"traceutil/trace.go:172","msg":"trace[1614113009] transaction","detail":"{read_only:false; response_revision:1255; number_of_response:1; }","duration":"184.918909ms","start":"2026-03-30T16:50:30.325470Z","end":"2026-03-30T16:50:30.510389Z","steps":["trace[1614113009] 'process raft request' (duration: 184.210792ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:30.510489Z","caller":"traceutil/trace.go:172","msg":"trace[774362290] transaction","detail":"{read_only:false; response_revision:1257; number_of_response:1; }","duration":"177.895947ms","start":"2026-03-30T16:50:30.332576Z","end":"2026-03-30T16:50:30.510472Z","steps":["trace[774362290] 'process raft request' (duration: 177.184529ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:30.854545Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"137.2538ms"} {"level":"info","ts":"2026-03-30T16:50:30.854649Z","caller":"traceutil/trace.go:172","msg":"trace[611938155] linearizableReadLoop","detail":"{readStateIndex:1490; appliedIndex:1492; }","duration":"341.591225ms","start":"2026-03-30T16:50:30.513024Z","end":"2026-03-30T16:50:30.854615Z","steps":["trace[611938155] 'read index received' (duration: 341.579174ms)","trace[611938155] 'applied index is now lower than readState.Index' (duration: 10.531µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:30.854711Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"137.427432ms"} {"level":"warn","ts":"2026-03-30T16:50:31.183462Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"670.423033ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-30T16:50:31.183523Z","caller":"traceutil/trace.go:172","msg":"trace[1255008567] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1260; }","duration":"670.487254ms","start":"2026-03-30T16:50:30.513019Z","end":"2026-03-30T16:50:31.183507Z","steps":["trace[1255008567] 'agreement among raft nodes before linearized reading' (duration: 341.798817ms)","trace[1255008567] 'range keys from in-memory index tree' (duration: 328.541286ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:31.183551Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:30.512993Z","time spent":"670.551454ms","remote":"127.0.0.1:55346","response type":"/etcdserverpb.KV/Range","request count":0,"request size":51,"response count":1,"response size":465,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:31.183714Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"128.832049ms"} {"level":"warn","ts":"2026-03-30T16:50:31.183778Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"128.90599ms"} {"level":"warn","ts":"2026-03-30T16:50:31.183969Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"328.805799ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:31.184192Z","caller":"traceutil/trace.go:172","msg":"trace[574441841] transaction","detail":"{read_only:false; response_revision:1261; number_of_response:1; }","duration":"671.522294ms","start":"2026-03-30T16:50:30.512628Z","end":"2026-03-30T16:50:31.184150Z","steps":["trace[574441841] 'process raft request' (duration: 342.483344ms)","trace[574441841] 'compare' (duration: 327.800488ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:31.184249Z","caller":"traceutil/trace.go:172","msg":"trace[873113572] transaction","detail":"{read_only:false; response_revision:1262; number_of_response:1; }","duration":"671.444403ms","start":"2026-03-30T16:50:30.512761Z","end":"2026-03-30T16:50:31.184205Z","steps":["trace[873113572] 'process raft request' (duration: 671.305072ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:31.184276Z","caller":"traceutil/trace.go:172","msg":"trace[2133586776] transaction","detail":"{read_only:false; response_revision:1263; number_of_response:1; }","duration":"671.074969ms","start":"2026-03-30T16:50:30.513185Z","end":"2026-03-30T16:50:31.184260Z","steps":["trace[2133586776] 'process raft request' (duration: 670.950618ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:31.184335Z","caller":"traceutil/trace.go:172","msg":"trace[1346382978] transaction","detail":"{read_only:false; response_revision:1266; number_of_response:1; }","duration":"660.55004ms","start":"2026-03-30T16:50:30.523771Z","end":"2026-03-30T16:50:31.184321Z","steps":["trace[1346382978] 'process raft request' (duration: 660.51985ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:31.184359Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:30.512609Z","time spent":"671.680996ms","remote":"127.0.0.1:55972","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":854,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:31.184366Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:30.512736Z","time spent":"671.557995ms","remote":"127.0.0.1:55972","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":854,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:31.184387Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:30.523750Z","time spent":"660.61259ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":751,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:31.184511Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:30.513162Z","time spent":"671.17884ms","remote":"127.0.0.1:55972","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":854,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:31.355107Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004351,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T16:50:31.403383Z","caller":"traceutil/trace.go:172","msg":"trace[837314349] linearizableReadLoop","detail":"{readStateIndex:1497; appliedIndex:1498; }","duration":"548.605117ms","start":"2026-03-30T16:50:30.854733Z","end":"2026-03-30T16:50:31.403339Z","steps":["trace[837314349] 'read index received' (duration: 548.590267ms)","trace[837314349] 'applied index is now lower than readState.Index' (duration: 12.77µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:31.505488Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004354,"received-request-id":10218277501228004351} {"level":"warn","ts":"2026-03-30T16:50:31.701984Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.177086575s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:31.702085Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"298.501403ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:31.702094Z","caller":"traceutil/trace.go:172","msg":"trace[1520646707] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1266; }","duration":"1.177193826s","start":"2026-03-30T16:50:30.524875Z","end":"2026-03-30T16:50:31.702068Z","steps":["trace[1520646707] 'agreement among raft nodes before linearized reading' (duration: 878.661292ms)","trace[1520646707] 'range keys from in-memory index tree' (duration: 298.385792ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:31.702599Z","caller":"traceutil/trace.go:172","msg":"trace[839092092] transaction","detail":"{read_only:false; response_revision:1269; number_of_response:1; }","duration":"490.584195ms","start":"2026-03-30T16:50:31.211992Z","end":"2026-03-30T16:50:31.702576Z","steps":["trace[839092092] 'process raft request' (duration: 490.506944ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:31.702661Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:30.524860Z","time spent":"1.177475158s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:31.702708Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.211969Z","time spent":"490.687396ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:50:31.702785Z","caller":"traceutil/trace.go:172","msg":"trace[80462042] transaction","detail":"{read_only:false; response_revision:1268; number_of_response:1; }","duration":"514.264371ms","start":"2026-03-30T16:50:31.188507Z","end":"2026-03-30T16:50:31.702772Z","steps":["trace[80462042] 'process raft request' (duration: 513.685684ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:31.702842Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.188455Z","time spent":"514.355002ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":709,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T16:50:31.702689Z","caller":"traceutil/trace.go:172","msg":"trace[1889031817] linearizableReadLoop","detail":"{readStateIndex:1500; appliedIndex:1498; }","duration":"299.195971ms","start":"2026-03-30T16:50:31.403475Z","end":"2026-03-30T16:50:31.702671Z","steps":["trace[1889031817] 'read index received' (duration: 102.071967ms)","trace[1889031817] 'applied index is now lower than readState.Index' (duration: 197.120994ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:31.702872Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"519.90827ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:31.702911Z","caller":"traceutil/trace.go:172","msg":"trace[582880401] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1269; }","duration":"519.95448ms","start":"2026-03-30T16:50:31.182945Z","end":"2026-03-30T16:50:31.702900Z","steps":["trace[582880401] 'agreement among raft nodes before linearized reading' (duration: 519.870979ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:31.702961Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"516.213331ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:3 size:387"} {"level":"info","ts":"2026-03-30T16:50:31.703002Z","caller":"traceutil/trace.go:172","msg":"trace[2108830659] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:3; response_revision:1269; }","duration":"516.262922ms","start":"2026-03-30T16:50:31.186730Z","end":"2026-03-30T16:50:31.702993Z","steps":["trace[2108830659] 'agreement among raft nodes before linearized reading' (duration: 516.117691ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:31.703025Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.186709Z","time spent":"516.312162ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":3,"response size":410,"request content":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" "} {"level":"warn","ts":"2026-03-30T16:50:31.780100Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"66.648513ms"} {"level":"warn","ts":"2026-03-30T16:50:31.780189Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"66.759034ms"} {"level":"warn","ts":"2026-03-30T16:50:32.203434Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004355,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:32.382377Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"402.29033ms"} {"level":"warn","ts":"2026-03-30T16:50:32.382446Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"402.25744ms"} {"level":"info","ts":"2026-03-30T16:50:32.382435Z","caller":"traceutil/trace.go:172","msg":"trace[704230333] linearizableReadLoop","detail":"{readStateIndex:1501; appliedIndex:1501; }","duration":"679.494729ms","start":"2026-03-30T16:50:31.702896Z","end":"2026-03-30T16:50:32.382390Z","steps":["trace[704230333] 'read index received' (duration: 679.486269ms)","trace[704230333] 'applied index is now lower than readState.Index' (duration: 7.07µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:32.382570Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"859.187193ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:32.382667Z","caller":"traceutil/trace.go:172","msg":"trace[128882031] range","detail":"{range_begin:/registry/resourceslices; range_end:; response_count:0; response_revision:1269; }","duration":"859.299954ms","start":"2026-03-30T16:50:31.523344Z","end":"2026-03-30T16:50:32.382644Z","steps":["trace[128882031] 'agreement among raft nodes before linearized reading' (duration: 859.112052ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:32.382723Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.523323Z","time spent":"859.387345ms","remote":"127.0.0.1:56032","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/resourceslices\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:32.383414Z","caller":"traceutil/trace.go:172","msg":"trace[1353270232] transaction","detail":"{read_only:false; response_revision:1272; number_of_response:1; }","duration":"677.238857ms","start":"2026-03-30T16:50:31.706153Z","end":"2026-03-30T16:50:32.383392Z","steps":["trace[1353270232] 'process raft request' (duration: 676.72698ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:32.383951Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.706131Z","time spent":"677.736922ms","remote":"127.0.0.1:55346","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":445,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:50:32.383995Z","caller":"traceutil/trace.go:172","msg":"trace[1444430449] transaction","detail":"{read_only:false; response_revision:1273; number_of_response:1; }","duration":"676.384498ms","start":"2026-03-30T16:50:31.707594Z","end":"2026-03-30T16:50:32.383978Z","steps":["trace[1444430449] 'process raft request' (duration: 675.358287ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:32.384105Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.707565Z","time spent":"676.477069ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":708,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:32.883219Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004358,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:32.910467Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"180.472109ms"} {"level":"warn","ts":"2026-03-30T16:50:32.910536Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"180.57651ms"} {"level":"info","ts":"2026-03-30T16:50:32.911147Z","caller":"traceutil/trace.go:172","msg":"trace[107572458] linearizableReadLoop","detail":"{readStateIndex:1505; appliedIndex:1505; }","duration":"528.639204ms","start":"2026-03-30T16:50:32.382479Z","end":"2026-03-30T16:50:32.911119Z","steps":["trace[107572458] 'read index received' (duration: 527.976507ms)","trace[107572458] 'applied index is now lower than readState.Index' (duration: 659.917µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:32.974894Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.270740573s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:32.975125Z","caller":"traceutil/trace.go:172","msg":"trace[1588344678] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1273; }","duration":"1.271085106s","start":"2026-03-30T16:50:31.703990Z","end":"2026-03-30T16:50:32.975075Z","steps":["trace[1588344678] 'agreement among raft nodes before linearized reading' (duration: 1.207238954s)","trace[1588344678] 'range keys from in-memory index tree' (duration: 63.464598ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:32.975179Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.194322951s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"warn","ts":"2026-03-30T16:50:32.975219Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.703897Z","time spent":"1.271274178s","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:50:32.975242Z","caller":"traceutil/trace.go:172","msg":"trace[720984318] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1273; }","duration":"1.194381632s","start":"2026-03-30T16:50:31.780848Z","end":"2026-03-30T16:50:32.975229Z","steps":["trace[720984318] 'agreement among raft nodes before linearized reading' (duration: 1.130309087s)","trace[720984318] 'range keys from in-memory index tree' (duration: 63.927282ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:32.975270Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.780828Z","time spent":"1.194436192s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:32.975411Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"754.691785ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:32.975442Z","caller":"traceutil/trace.go:172","msg":"trace[1452602777] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1273; }","duration":"754.721546ms","start":"2026-03-30T16:50:32.220713Z","end":"2026-03-30T16:50:32.975435Z","steps":["trace[1452602777] 'agreement among raft nodes before linearized reading' (duration: 690.47035ms)","trace[1452602777] 'range keys from in-memory index tree' (duration: 64.216695ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:32.975662Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.235304035s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:19145"} {"level":"info","ts":"2026-03-30T16:50:32.975707Z","caller":"traceutil/trace.go:172","msg":"trace[2041006305] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:1273; }","duration":"1.235351055s","start":"2026-03-30T16:50:31.740344Z","end":"2026-03-30T16:50:32.975695Z","steps":["trace[2041006305] 'agreement among raft nodes before linearized reading' (duration: 1.170844537s)","trace[2041006305] 'range keys from in-memory index tree' (duration: 64.339267ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:32.975687Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"445.640975ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T16:50:32.975737Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:31.740327Z","time spent":"1.235402177s","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":19168,"request content":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:33.165596Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"52.682834ms"} {"level":"warn","ts":"2026-03-30T16:50:33.165665Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"52.770225ms"} {"level":"info","ts":"2026-03-30T16:50:33.235227Z","caller":"traceutil/trace.go:172","msg":"trace[1902179459] linearizableReadLoop","detail":"{readStateIndex:1506; appliedIndex:1506; }","duration":"323.974932ms","start":"2026-03-30T16:50:32.911222Z","end":"2026-03-30T16:50:33.235197Z","steps":["trace[1902179459] 'read index received' (duration: 323.961352ms)","trace[1902179459] 'applied index is now lower than readState.Index' (duration: 11.67µs)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:33.235478Z","caller":"traceutil/trace.go:172","msg":"trace[111926463] transaction","detail":"{read_only:false; response_revision:1277; number_of_response:1; }","duration":"246.283637ms","start":"2026-03-30T16:50:32.989177Z","end":"2026-03-30T16:50:33.235461Z","steps":["trace[111926463] 'process raft request' (duration: 246.233257ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:33.235508Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"848.502373ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:498"} {"level":"info","ts":"2026-03-30T16:50:33.235555Z","caller":"traceutil/trace.go:172","msg":"trace[696236930] range","detail":"{range_begin:/registry/endpointslices/default/kubernetes; range_end:; response_count:1; response_revision:1275; }","duration":"848.568724ms","start":"2026-03-30T16:50:32.386976Z","end":"2026-03-30T16:50:33.235545Z","steps":["trace[696236930] 'agreement among raft nodes before linearized reading' (duration: 848.373002ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:33.235556Z","caller":"traceutil/trace.go:172","msg":"trace[979576937] transaction","detail":"{read_only:false; response_revision:1276; number_of_response:1; }","duration":"249.574791ms","start":"2026-03-30T16:50:32.985965Z","end":"2026-03-30T16:50:33.235540Z","steps":["trace[979576937] 'process raft request' (duration: 249.412589ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:33.235591Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:32.386953Z","time spent":"848.629334ms","remote":"127.0.0.1:55526","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":521,"request content":"key:\"/registry/endpointslices/default/kubernetes\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:33.235711Z","caller":"traceutil/trace.go:172","msg":"trace[1175262758] transaction","detail":"{read_only:false; response_revision:1275; number_of_response:1; }","duration":"709.925838ms","start":"2026-03-30T16:50:32.525762Z","end":"2026-03-30T16:50:33.235688Z","steps":["trace[1175262758] 'process raft request' (duration: 709.507894ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:33.235469Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"777.802371ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:33.235861Z","caller":"traceutil/trace.go:172","msg":"trace[1225806020] range","detail":"{range_begin:/registry/endpointslices; range_end:; response_count:0; response_revision:1275; }","duration":"778.240135ms","start":"2026-03-30T16:50:32.457610Z","end":"2026-03-30T16:50:33.235850Z","steps":["trace[1225806020] 'agreement among raft nodes before linearized reading' (duration: 777.73001ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:33.235895Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:32.457594Z","time spent":"778.290676ms","remote":"127.0.0.1:55526","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/endpointslices\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:33.235890Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:32.525739Z","time spent":"710.083201ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:50:33.348980Z","caller":"traceutil/trace.go:172","msg":"trace[903040898] linearizableReadLoop","detail":"{readStateIndex:1509; appliedIndex:1510; }","duration":"113.478213ms","start":"2026-03-30T16:50:33.235306Z","end":"2026-03-30T16:50:33.348784Z","steps":["trace[903040898] 'read index received' (duration: 113.470993ms)","trace[903040898] 'applied index is now lower than readState.Index' (duration: 5.57µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:33.348986Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"138.373931ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:33.349123Z","caller":"traceutil/trace.go:172","msg":"trace[1373300040] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1278; }","duration":"138.525193ms","start":"2026-03-30T16:50:33.210582Z","end":"2026-03-30T16:50:33.349107Z","steps":["trace[1373300040] 'agreement among raft nodes before linearized reading' (duration: 138.321461ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:33.349081Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"367.313649ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:33.349650Z","caller":"traceutil/trace.go:172","msg":"trace[1966036584] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1278; }","duration":"367.731883ms","start":"2026-03-30T16:50:32.981744Z","end":"2026-03-30T16:50:33.349476Z","steps":["trace[1966036584] 'agreement among raft nodes before linearized reading' (duration: 367.288799ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:33.349742Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:32.981729Z","time spent":"368.002605ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:50:33.352848Z","caller":"traceutil/trace.go:172","msg":"trace[2127596929] transaction","detail":"{read_only:false; response_revision:1280; number_of_response:1; }","duration":"113.394343ms","start":"2026-03-30T16:50:33.239435Z","end":"2026-03-30T16:50:33.352830Z","steps":["trace[2127596929] 'process raft request' (duration: 113.264582ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:33.390028Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"100.388588ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:33.390108Z","caller":"traceutil/trace.go:172","msg":"trace[1852809089] range","detail":"{range_begin:/registry/rolebindings; range_end:; response_count:0; response_revision:1280; }","duration":"100.490159ms","start":"2026-03-30T16:50:33.289603Z","end":"2026-03-30T16:50:33.390093Z","steps":["trace[1852809089] 'agreement among raft nodes before linearized reading' (duration: 66.468738ms)","trace[1852809089] 'range keys from in-memory index tree' (duration: 33.8946ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:33.390159Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"149.219103ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:19318"} {"level":"info","ts":"2026-03-30T16:50:33.390235Z","caller":"traceutil/trace.go:172","msg":"trace[359503715] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:1280; }","duration":"149.304073ms","start":"2026-03-30T16:50:33.240911Z","end":"2026-03-30T16:50:33.390215Z","steps":["trace[359503715] 'agreement among raft nodes before linearized reading' (duration: 115.180471ms)","trace[359503715] 'range keys from in-memory index tree' (duration: 33.938831ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:33.765363Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"114.823276ms"} {"level":"warn","ts":"2026-03-30T16:50:33.765431Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"114.899697ms"} {"level":"info","ts":"2026-03-30T16:50:33.766717Z","caller":"traceutil/trace.go:172","msg":"trace[1907124881] linearizableReadLoop","detail":"{readStateIndex:1519; appliedIndex:1519; }","duration":"237.504446ms","start":"2026-03-30T16:50:33.529188Z","end":"2026-03-30T16:50:33.766692Z","steps":["trace[1907124881] 'read index received' (duration: 237.497075ms)","trace[1907124881] 'applied index is now lower than readState.Index' (duration: 6.04µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:34.116156Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"102.952363ms"} {"level":"warn","ts":"2026-03-30T16:50:34.116225Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"103.028873ms"} {"level":"warn","ts":"2026-03-30T16:50:34.125347Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"636.557662ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattachments\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:34.125413Z","caller":"traceutil/trace.go:172","msg":"trace[1802410303] range","detail":"{range_begin:/registry/volumeattachments; range_end:; response_count:0; response_revision:1284; }","duration":"636.629773ms","start":"2026-03-30T16:50:33.488764Z","end":"2026-03-30T16:50:34.125394Z","steps":["trace[1802410303] 'agreement among raft nodes before linearized reading' (duration: 278.050385ms)","trace[1802410303] 'range keys from in-memory index tree' (duration: 358.483677ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:34.125463Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:33.488741Z","time spent":"636.698813ms","remote":"127.0.0.1:55718","response type":"/etcdserverpb.KV/Range","request count":0,"request size":31,"response count":0,"response size":28,"request content":"key:\"/registry/volumeattachments\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:34.125531Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"359.843071ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:34.125705Z","caller":"traceutil/trace.go:172","msg":"trace[1739455222] transaction","detail":"{read_only:false; response_revision:1285; number_of_response:1; }","duration":"657.58445ms","start":"2026-03-30T16:50:33.468099Z","end":"2026-03-30T16:50:34.125684Z","steps":["trace[1739455222] 'process raft request' (duration: 297.532647ms)","trace[1739455222] 'compare' (duration: 359.584078ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:34.125721Z","caller":"traceutil/trace.go:172","msg":"trace[493326533] linearizableReadLoop","detail":"{readStateIndex:1521; appliedIndex:1519; }","duration":"358.944782ms","start":"2026-03-30T16:50:33.766765Z","end":"2026-03-30T16:50:34.125710Z","steps":["trace[493326533] 'read index received' (duration: 760.279µs)","trace[493326533] 'applied index is now lower than readState.Index' (duration: 358.183743ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:34.125759Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:33.468072Z","time spent":"657.66428ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":721,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:34.125774Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"546.285328ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/daemonsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:34.125807Z","caller":"traceutil/trace.go:172","msg":"trace[1835304507] range","detail":"{range_begin:/registry/daemonsets; range_end:; response_count:0; response_revision:1287; }","duration":"546.319529ms","start":"2026-03-30T16:50:33.579479Z","end":"2026-03-30T16:50:34.125798Z","steps":["trace[1835304507] 'agreement among raft nodes before linearized reading' (duration: 546.258838ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:34.125833Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:33.579456Z","time spent":"546.37068ms","remote":"127.0.0.1:55864","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/daemonsets\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:34.125867Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"562.247374ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T16:50:34.125894Z","caller":"traceutil/trace.go:172","msg":"trace[1970219019] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1287; }","duration":"562.280225ms","start":"2026-03-30T16:50:33.563605Z","end":"2026-03-30T16:50:34.125885Z","steps":["trace[1970219019] 'agreement among raft nodes before linearized reading' (duration: 562.158792ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:34.125921Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:33.563579Z","time spent":"562.334815ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:34.379239Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"55.754532ms"} {"level":"warn","ts":"2026-03-30T16:50:34.379473Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"55.991224ms"} {"level":"warn","ts":"2026-03-30T16:50:34.680127Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004375,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:34.863183Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"249.91287ms"} {"level":"warn","ts":"2026-03-30T16:50:34.863253Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"249.989291ms"} {"level":"info","ts":"2026-03-30T16:50:34.863249Z","caller":"traceutil/trace.go:172","msg":"trace[1744408468] linearizableReadLoop","detail":"{readStateIndex:1522; appliedIndex:1523; }","duration":"684.143309ms","start":"2026-03-30T16:50:34.179079Z","end":"2026-03-30T16:50:34.863222Z","steps":["trace[1744408468] 'read index received' (duration: 684.134119ms)","trace[1744408468] 'applied index is now lower than readState.Index' (duration: 7.28µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:34.863639Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"684.380192ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:34.863763Z","caller":"traceutil/trace.go:172","msg":"trace[1660384659] transaction","detail":"{read_only:false; response_revision:1290; number_of_response:1; }","duration":"450.212133ms","start":"2026-03-30T16:50:34.413534Z","end":"2026-03-30T16:50:34.863746Z","steps":["trace[1660384659] 'process raft request' (duration: 450.167442ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:34.863831Z","caller":"traceutil/trace.go:172","msg":"trace[1408792898] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1288; }","duration":"684.741945ms","start":"2026-03-30T16:50:34.179073Z","end":"2026-03-30T16:50:34.863815Z","steps":["trace[1408792898] 'agreement among raft nodes before linearized reading' (duration: 684.354741ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:34.863865Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:34.413505Z","time spent":"450.305643ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":720,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:35.204895Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"141.617657ms"} {"level":"warn","ts":"2026-03-30T16:50:35.204951Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"141.681108ms"} {"level":"warn","ts":"2026-03-30T16:50:35.364140Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004377,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:35.542774Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"129.40636ms"} {"level":"warn","ts":"2026-03-30T16:50:35.542854Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"129.495901ms"} {"level":"warn","ts":"2026-03-30T16:50:35.614210Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"435.688408ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:35.614508Z","caller":"traceutil/trace.go:172","msg":"trace[893968547] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1290; }","duration":"436.037351ms","start":"2026-03-30T16:50:35.178457Z","end":"2026-03-30T16:50:35.614495Z","steps":["trace[893968547] 'range keys from in-memory index tree' (duration: 435.620317ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:35.614385Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"217.601736ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:35.614982Z","caller":"traceutil/trace.go:172","msg":"trace[559675304] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1290; }","duration":"218.211132ms","start":"2026-03-30T16:50:35.396760Z","end":"2026-03-30T16:50:35.614972Z","steps":["trace[559675304] 'range keys from in-memory index tree' (duration: 217.503436ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:35.691476Z","caller":"traceutil/trace.go:172","msg":"trace[1561356087] transaction","detail":"{read_only:false; response_revision:1293; number_of_response:1; }","duration":"443.454067ms","start":"2026-03-30T16:50:35.247999Z","end":"2026-03-30T16:50:35.691453Z","steps":["trace[1561356087] 'process raft request' (duration: 443.338806ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:35.691530Z","caller":"traceutil/trace.go:172","msg":"trace[959401305] linearizableReadLoop","detail":"{readStateIndex:1525; appliedIndex:1528; }","duration":"828.186089ms","start":"2026-03-30T16:50:34.863318Z","end":"2026-03-30T16:50:35.691504Z","steps":["trace[959401305] 'read index received' (duration: 828.178019ms)","trace[959401305] 'applied index is now lower than readState.Index' (duration: 6.36µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:35.691647Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:35.247978Z","time spent":"443.590968ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":466,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:50:35.691676Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.358756212s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:35.691708Z","caller":"traceutil/trace.go:172","msg":"trace[483274992] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1293; }","duration":"1.358799892s","start":"2026-03-30T16:50:34.332898Z","end":"2026-03-30T16:50:35.691698Z","steps":["trace[483274992] 'agreement among raft nodes before linearized reading' (duration: 1.358728892s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:35.691747Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:34.332869Z","time spent":"1.358871614s","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:36.008506Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"165.366497ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:36.008698Z","caller":"traceutil/trace.go:172","msg":"trace[1457919207] transaction","detail":"{read_only:false; response_revision:1295; number_of_response:1; }","duration":"383.288948ms","start":"2026-03-30T16:50:35.625398Z","end":"2026-03-30T16:50:36.008687Z","steps":["trace[1457919207] 'process raft request' (duration: 383.204987ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:36.008913Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:35.625374Z","time spent":"383.51495ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":759,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:36.192353Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004380,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:36.210853Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"167.941507ms"} {"level":"warn","ts":"2026-03-30T16:50:36.210997Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"168.100798ms"} {"level":"info","ts":"2026-03-30T16:50:36.210930Z","caller":"traceutil/trace.go:172","msg":"trace[1398239036] linearizableReadLoop","detail":"{readStateIndex:1528; appliedIndex:1530; }","duration":"519.309149ms","start":"2026-03-30T16:50:35.691589Z","end":"2026-03-30T16:50:36.210899Z","steps":["trace[1398239036] 'read index received' (duration: 519.297229ms)","trace[1398239036] 'applied index is now lower than readState.Index' (duration: 9.89µs)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:36.312986Z","caller":"traceutil/trace.go:172","msg":"trace[1321749868] linearizableReadLoop","detail":"{readStateIndex:1530; appliedIndex:1530; }","duration":"101.88251ms","start":"2026-03-30T16:50:36.211079Z","end":"2026-03-30T16:50:36.312962Z","steps":["trace[1321749868] 'read index received' (duration: 101.87342ms)","trace[1321749868] 'applied index is now lower than readState.Index' (duration: 7.53µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.735033Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.843268971s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaimtemplates\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:36.735105Z","caller":"traceutil/trace.go:172","msg":"trace[619224270] range","detail":"{range_begin:/registry/resourceclaimtemplates; range_end:; response_count:0; response_revision:1295; }","duration":"1.843362382s","start":"2026-03-30T16:50:34.891729Z","end":"2026-03-30T16:50:36.735092Z","steps":["trace[619224270] 'agreement among raft nodes before linearized reading' (duration: 1.319262676s)","trace[619224270] 'range keys from in-memory index tree' (duration: 523.973225ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.735132Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:34.891711Z","time spent":"1.843415752s","remote":"127.0.0.1:56026","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":28,"request content":"key:\"/registry/resourceclaimtemplates\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:36.735172Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.589325508s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T16:50:36.735212Z","caller":"traceutil/trace.go:172","msg":"trace[1491137768] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1295; }","duration":"1.589355289s","start":"2026-03-30T16:50:35.145840Z","end":"2026-03-30T16:50:36.735195Z","steps":["trace[1491137768] 'agreement among raft nodes before linearized reading' (duration: 1.065176883s)","trace[1491137768] 'range keys from in-memory index tree' (duration: 524.067595ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.735247Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:35.145816Z","time spent":"1.58942328s","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:36.735234Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.042456494s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:36.735309Z","caller":"traceutil/trace.go:172","msg":"trace[1309599886] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1295; }","duration":"1.042535915s","start":"2026-03-30T16:50:35.692754Z","end":"2026-03-30T16:50:36.735290Z","steps":["trace[1309599886] 'agreement among raft nodes before linearized reading' (duration: 620.298299ms)","trace[1309599886] 'range keys from in-memory index tree' (duration: 422.130255ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.735287Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.546221954s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingwebhookconfigurations\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:36.735371Z","caller":"traceutil/trace.go:172","msg":"trace[948300997] range","detail":"{range_begin:/registry/validatingwebhookconfigurations; range_end:; response_count:0; response_revision:1295; }","duration":"1.546322985s","start":"2026-03-30T16:50:35.189028Z","end":"2026-03-30T16:50:36.735351Z","steps":["trace[948300997] 'agreement among raft nodes before linearized reading' (duration: 1.021892797s)","trace[948300997] 'range keys from in-memory index tree' (duration: 524.247027ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.735402Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.120854344s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:36.735422Z","caller":"traceutil/trace.go:172","msg":"trace[1372671769] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1295; }","duration":"1.120867784s","start":"2026-03-30T16:50:35.614544Z","end":"2026-03-30T16:50:36.735412Z","steps":["trace[1372671769] 'agreement among raft nodes before linearized reading' (duration: 596.453925ms)","trace[1372671769] 'range keys from in-memory index tree' (duration: 524.397179ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.735427Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:35.189016Z","time spent":"1.546401676s","remote":"127.0.0.1:55906","response type":"/etcdserverpb.KV/Range","request count":0,"request size":45,"response count":0,"response size":28,"request content":"key:\"/registry/validatingwebhookconfigurations\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:36.735430Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:35.692733Z","time spent":"1.042608695s","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:36.735443Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.743873093s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/statefulsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T16:50:36.735039Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.622636403s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:36.735475Z","caller":"traceutil/trace.go:172","msg":"trace[1515460740] range","detail":"{range_begin:/registry/statefulsets; range_end:; response_count:0; response_revision:1295; }","duration":"1.743903953s","start":"2026-03-30T16:50:34.991561Z","end":"2026-03-30T16:50:36.735465Z","steps":["trace[1515460740] 'agreement among raft nodes before linearized reading' (duration: 1.219453025s)","trace[1515460740] 'range keys from in-memory index tree' (duration: 524.406248ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.735545Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:34.991541Z","time spent":"1.743966725s","remote":"127.0.0.1:55854","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/statefulsets\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:36.735515Z","caller":"traceutil/trace.go:172","msg":"trace[722866457] range","detail":"{range_begin:/registry/minions; range_end:; response_count:0; response_revision:1295; }","duration":"1.623114858s","start":"2026-03-30T16:50:35.112379Z","end":"2026-03-30T16:50:36.735494Z","steps":["trace[722866457] 'agreement among raft nodes before linearized reading' (duration: 1.098626458s)","trace[722866457] 'range keys from in-memory index tree' (duration: 523.990355ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.735384Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.111028473s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:19411"} {"level":"warn","ts":"2026-03-30T16:50:36.735877Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"134.81984ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:36.735912Z","caller":"traceutil/trace.go:172","msg":"trace[872840655] range","detail":"{range_begin:/registry/serviceaccounts; range_end:; response_count:0; response_revision:1295; }","duration":"134.856781ms","start":"2026-03-30T16:50:36.601046Z","end":"2026-03-30T16:50:36.735902Z","steps":["trace[872840655] 'range keys from in-memory index tree' (duration: 121.628854ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:36.736092Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"131.336764ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T16:50:36.735651Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:35.112357Z","time spent":"1.623277029s","remote":"127.0.0.1:55364","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":28,"request content":"key:\"/registry/minions\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:36.735705Z","caller":"traceutil/trace.go:172","msg":"trace[2136539489] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:1295; }","duration":"1.111342076s","start":"2026-03-30T16:50:35.624351Z","end":"2026-03-30T16:50:36.735693Z","steps":["trace[2136539489] 'agreement among raft nodes before linearized reading' (duration: 586.630274ms)","trace[2136539489] 'range keys from in-memory index tree' (duration: 524.333568ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:36.736912Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:35.624333Z","time spent":"1.112506538s","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":19434,"request content":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:37.158782Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"422.313034ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T16:50:37.158880Z","caller":"traceutil/trace.go:172","msg":"trace[970245827] linearizableReadLoop","detail":"{readStateIndex:1534; appliedIndex:1530; }","duration":"507.786615ms","start":"2026-03-30T16:50:36.651078Z","end":"2026-03-30T16:50:37.158865Z","steps":["trace[970245827] 'read index received' (duration: 867.389µs)","trace[970245827] 'applied index is now lower than readState.Index' (duration: 506.918036ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:37.158869Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"420.848718ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:37.158965Z","caller":"traceutil/trace.go:172","msg":"trace[252767776] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1298; }","duration":"420.966409ms","start":"2026-03-30T16:50:36.737976Z","end":"2026-03-30T16:50:37.158942Z","steps":["trace[252767776] 'range keys from in-memory index tree' (duration: 420.796798ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:37.159335Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"508.24341ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T16:50:37.159378Z","caller":"traceutil/trace.go:172","msg":"trace[1004287141] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1298; }","duration":"508.2945ms","start":"2026-03-30T16:50:36.651070Z","end":"2026-03-30T16:50:37.159365Z","steps":["trace[1004287141] 'agreement among raft nodes before linearized reading' (duration: 508.033888ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:37.159416Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:36.651049Z","time spent":"508.356581ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:50:37.291836Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"378.640686ms"} {"level":"warn","ts":"2026-03-30T16:50:37.291888Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"378.696296ms"} {"level":"info","ts":"2026-03-30T16:50:37.292225Z","caller":"traceutil/trace.go:172","msg":"trace[1259248526] transaction","detail":"{read_only:false; response_revision:1299; number_of_response:1; }","duration":"525.164114ms","start":"2026-03-30T16:50:36.767041Z","end":"2026-03-30T16:50:37.292205Z","steps":["trace[1259248526] 'process raft request' (duration: 524.907662ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:37.292443Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:36.767017Z","time spent":"525.306066ms","remote":"127.0.0.1:55396","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":19655,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:50:37.362831Z","caller":"traceutil/trace.go:172","msg":"trace[866621946] linearizableReadLoop","detail":"{readStateIndex:1534; appliedIndex:1535; }","duration":"203.903692ms","start":"2026-03-30T16:50:37.158904Z","end":"2026-03-30T16:50:37.362808Z","steps":["trace[866621946] 'read index received' (duration: 203.895802ms)","trace[866621946] 'applied index is now lower than readState.Index' (duration: 6.59µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:37.491820Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"749.067781ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:37.491873Z","caller":"traceutil/trace.go:172","msg":"trace[1597712078] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1299; }","duration":"749.127401ms","start":"2026-03-30T16:50:36.742735Z","end":"2026-03-30T16:50:37.491862Z","steps":["trace[1597712078] 'agreement among raft nodes before linearized reading' (duration: 620.255244ms)","trace[1597712078] 'range keys from in-memory index tree' (duration: 128.771757ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:37.491895Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:36.742713Z","time spent":"749.176812ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:37.492680Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"129.617385ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:37.493287Z","caller":"traceutil/trace.go:172","msg":"trace[349750120] transaction","detail":"{read_only:false; response_revision:1300; number_of_response:1; }","duration":"726.011364ms","start":"2026-03-30T16:50:36.767266Z","end":"2026-03-30T16:50:37.493278Z","steps":["trace[349750120] 'process raft request' (duration: 595.740272ms)","trace[349750120] 'compare' (duration: 129.08904ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:37.493329Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:36.767246Z","time spent":"726.061515ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:50:37.496325Z","caller":"traceutil/trace.go:172","msg":"trace[337137160] linearizableReadLoop","detail":"{readStateIndex:1537; appliedIndex:1537; }","duration":"133.373625ms","start":"2026-03-30T16:50:37.362929Z","end":"2026-03-30T16:50:37.496303Z","steps":["trace[337137160] 'read index received' (duration: 133.366445ms)","trace[337137160] 'applied index is now lower than readState.Index' (duration: 5.91µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:37.496512Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"269.221084ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T16:50:37.496576Z","caller":"traceutil/trace.go:172","msg":"trace[637101583] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1301; }","duration":"269.298275ms","start":"2026-03-30T16:50:37.227264Z","end":"2026-03-30T16:50:37.496562Z","steps":["trace[637101583] 'agreement among raft nodes before linearized reading' (duration: 269.083313ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:37.496546Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"337.462448ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:37.496734Z","caller":"traceutil/trace.go:172","msg":"trace[832124708] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1301; }","duration":"337.681851ms","start":"2026-03-30T16:50:37.159022Z","end":"2026-03-30T16:50:37.496704Z","steps":["trace[832124708] 'agreement among raft nodes before linearized reading' (duration: 337.390318ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:37.715414Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"13.349688ms"} {"level":"warn","ts":"2026-03-30T16:50:37.715484Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"13.429268ms"} {"level":"warn","ts":"2026-03-30T16:50:38.070567Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004391,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:38.123715Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"110.370267ms"} {"level":"warn","ts":"2026-03-30T16:50:38.123783Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"110.444797ms"} {"level":"info","ts":"2026-03-30T16:50:38.123761Z","caller":"traceutil/trace.go:172","msg":"trace[1982983870] linearizableReadLoop","detail":"{readStateIndex:1538; appliedIndex:1539; }","duration":"553.927897ms","start":"2026-03-30T16:50:37.569803Z","end":"2026-03-30T16:50:38.123731Z","steps":["trace[1982983870] 'read index received' (duration: 553.914837ms)","trace[1982983870] 'applied index is now lower than readState.Index' (duration: 11.21µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:38.201653Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"682.394268ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.227\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-30T16:50:38.201715Z","caller":"traceutil/trace.go:172","msg":"trace[270298810] range","detail":"{range_begin:/registry/masterleases/199.204.45.227; range_end:; response_count:1; response_revision:1302; }","duration":"682.464558ms","start":"2026-03-30T16:50:37.519237Z","end":"2026-03-30T16:50:38.201702Z","steps":["trace[270298810] 'agreement among raft nodes before linearized reading' (duration: 604.575838ms)","trace[270298810] 'range keys from in-memory index tree' (duration: 77.732399ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:38.201748Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:37.519205Z","time spent":"682.53735ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.227\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:38.201844Z","caller":"traceutil/trace.go:172","msg":"trace[1659995262] transaction","detail":"{read_only:false; response_revision:1304; number_of_response:1; }","duration":"504.71867ms","start":"2026-03-30T16:50:37.697107Z","end":"2026-03-30T16:50:38.201826Z","steps":["trace[1659995262] 'process raft request' (duration: 504.622508ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:38.201980Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:37.697065Z","time spent":"504.855481ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:50:38.239786Z","caller":"traceutil/trace.go:172","msg":"trace[952173579] linearizableReadLoop","detail":"{readStateIndex:1542; appliedIndex:1543; }","duration":"115.838321ms","start":"2026-03-30T16:50:38.123929Z","end":"2026-03-30T16:50:38.239767Z","steps":["trace[952173579] 'read index received' (duration: 115.834291ms)","trace[952173579] 'applied index is now lower than readState.Index' (duration: 3.56µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:38.239898Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"669.91754ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:38.239920Z","caller":"traceutil/trace.go:172","msg":"trace[1934421458] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1306; }","duration":"669.96224ms","start":"2026-03-30T16:50:37.569951Z","end":"2026-03-30T16:50:38.239913Z","steps":["trace[1934421458] 'agreement among raft nodes before linearized reading' (duration: 669.90361ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:38.741892Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004396,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:50:38.784698Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"327.38567ms"} {"level":"warn","ts":"2026-03-30T16:50:38.784754Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"327.448051ms"} {"level":"warn","ts":"2026-03-30T16:50:38.785039Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:38.203411Z","time spent":"581.623958ms","remote":"127.0.0.1:55086","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-30T16:50:38.897915Z","caller":"traceutil/trace.go:172","msg":"trace[1536558900] linearizableReadLoop","detail":"{readStateIndex:1543; appliedIndex:1544; }","duration":"656.904574ms","start":"2026-03-30T16:50:38.240987Z","end":"2026-03-30T16:50:38.897891Z","steps":["trace[1536558900] 'read index received' (duration: 656.895644ms)","trace[1536558900] 'applied index is now lower than readState.Index' (duration: 7.83µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:38.925686Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"684.67447ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:38.925758Z","caller":"traceutil/trace.go:172","msg":"trace[1923596887] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1306; }","duration":"684.75838ms","start":"2026-03-30T16:50:38.240982Z","end":"2026-03-30T16:50:38.925740Z","steps":["trace[1923596887] 'agreement among raft nodes before linearized reading' (duration: 657.138546ms)","trace[1923596887] 'range keys from in-memory index tree' (duration: 27.511963ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T16:50:38.925981Z","caller":"traceutil/trace.go:172","msg":"trace[1252946041] transaction","detail":"{read_only:false; response_revision:1307; number_of_response:1; }","duration":"678.382614ms","start":"2026-03-30T16:50:38.247582Z","end":"2026-03-30T16:50:38.925965Z","steps":["trace[1252946041] 'process raft request' (duration: 650.608648ms)","trace[1252946041] 'compare' (duration: 27.431752ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:38.926076Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:38.247558Z","time spent":"678.465415ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":725,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:38.926328Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"584.329776ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:38.926355Z","caller":"traceutil/trace.go:172","msg":"trace[783473299] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1308; }","duration":"584.366316ms","start":"2026-03-30T16:50:38.341983Z","end":"2026-03-30T16:50:38.926349Z","steps":["trace[783473299] 'agreement among raft nodes before linearized reading' (duration: 584.314146ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:38.926374Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:38.341841Z","time spent":"584.528428ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:50:38.926354Z","caller":"traceutil/trace.go:172","msg":"trace[1670065456] transaction","detail":"{read_only:false; response_revision:1308; number_of_response:1; }","duration":"139.86084ms","start":"2026-03-30T16:50:38.786486Z","end":"2026-03-30T16:50:38.926346Z","steps":["trace[1670065456] 'process raft request' (duration: 139.700119ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:50:39.209843Z","caller":"traceutil/trace.go:172","msg":"trace[1904839293] linearizableReadLoop","detail":"{readStateIndex:1549; appliedIndex:1551; }","duration":"222.078203ms","start":"2026-03-30T16:50:38.987737Z","end":"2026-03-30T16:50:39.209815Z","steps":["trace[1904839293] 'read index received' (duration: 222.066003ms)","trace[1904839293] 'applied index is now lower than readState.Index' (duration: 10.41µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:39.421939Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"9.174772ms"} {"level":"warn","ts":"2026-03-30T16:50:39.422006Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"9.254183ms"} {"level":"warn","ts":"2026-03-30T16:50:39.516423Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"338.090248ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:39.516522Z","caller":"traceutil/trace.go:172","msg":"trace[493678018] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1312; }","duration":"338.195039ms","start":"2026-03-30T16:50:39.178308Z","end":"2026-03-30T16:50:39.516503Z","steps":["trace[493678018] 'range keys from in-memory index tree' (duration: 338.042737ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:39.516904Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"587.704456ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:460"} {"level":"info","ts":"2026-03-30T16:50:39.516979Z","caller":"traceutil/trace.go:172","msg":"trace[601832074] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1312; }","duration":"587.829157ms","start":"2026-03-30T16:50:38.929137Z","end":"2026-03-30T16:50:39.516966Z","steps":["trace[601832074] 'agreement among raft nodes before linearized reading' (duration: 280.777908ms)","trace[601832074] 'range keys from in-memory index tree' (duration: 306.807926ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:39.516982Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"306.999308ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T16:50:39.517032Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:38.928975Z","time spent":"588.048318ms","remote":"127.0.0.1:55346","response type":"/etcdserverpb.KV/Range","request count":0,"request size":51,"response count":1,"response size":483,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 "} {"level":"info","ts":"2026-03-30T16:50:39.517284Z","caller":"traceutil/trace.go:172","msg":"trace[1159454552] transaction","detail":"{read_only:false; response_revision:1314; number_of_response:1; }","duration":"470.54158ms","start":"2026-03-30T16:50:39.046731Z","end":"2026-03-30T16:50:39.517273Z","steps":["trace[1159454552] 'process raft request' (duration: 470.324168ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:39.517484Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:39.046711Z","time spent":"470.60246ms","remote":"127.0.0.1:55168","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":724,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T16:50:40.739474Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"405.435475ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:40.739549Z","caller":"traceutil/trace.go:172","msg":"trace[48073705] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1371; }","duration":"405.524106ms","start":"2026-03-30T16:50:40.334012Z","end":"2026-03-30T16:50:40.739536Z","steps":["trace[48073705] 'range keys from in-memory index tree' (duration: 404.802509ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:40.739582Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:50:40.334004Z","time spent":"405.570856ms","remote":"127.0.0.1:57720","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:50:40.739826Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"120.884492ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:50:40.844228Z","caller":"traceutil/trace.go:172","msg":"trace[1579065455] linearizableReadLoop","detail":"{readStateIndex:1613; appliedIndex:1614; }","duration":"142.07299ms","start":"2026-03-30T16:50:40.702130Z","end":"2026-03-30T16:50:40.844203Z","steps":["trace[1579065455] 'read index received' (duration: 142.06313ms)","trace[1579065455] 'applied index is now lower than readState.Index' (duration: 8.45µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:50:40.847263Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"145.089571ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-gp9q8\" limit:1 ","response":"range_response_count:1 size:19904"} {"level":"info","ts":"2026-03-30T16:50:40.847321Z","caller":"traceutil/trace.go:172","msg":"trace[2146169127] range","detail":"{range_begin:/registry/pods/kube-system/cilium-gp9q8; range_end:; response_count:1; response_revision:1373; }","duration":"145.183162ms","start":"2026-03-30T16:50:40.702125Z","end":"2026-03-30T16:50:40.847309Z","steps":["trace[2146169127] 'agreement among raft nodes before linearized reading' (duration: 142.208701ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:41.959637Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39820","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T16:50:44.383152Z","caller":"traceutil/trace.go:172","msg":"trace[2124286883] transaction","detail":"{read_only:false; response_revision:1412; number_of_response:1; }","duration":"141.107034ms","start":"2026-03-30T16:50:44.242020Z","end":"2026-03-30T16:50:44.383127Z","steps":["trace[2124286883] 'process raft request' (duration: 140.920732ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:44.593790Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"104.547331ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumnodes/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:44.593862Z","caller":"traceutil/trace.go:172","msg":"trace[696543249] range","detail":"{range_begin:/registry/cilium.io/ciliumnodes/controller-2; range_end:; response_count:0; response_revision:1413; }","duration":"104.638602ms","start":"2026-03-30T16:50:44.489210Z","end":"2026-03-30T16:50:44.593848Z","steps":["trace[696543249] 'range keys from in-memory index tree' (duration: 103.48247ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:44.593988Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"104.44858ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumnodes/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:44.594066Z","caller":"traceutil/trace.go:172","msg":"trace[66076246] range","detail":"{range_begin:/registry/cilium.io/ciliumnodes/controller-2; range_end:; response_count:0; response_revision:1413; }","duration":"104.532331ms","start":"2026-03-30T16:50:44.489516Z","end":"2026-03-30T16:50:44.594049Z","steps":["trace[66076246] 'range keys from in-memory index tree' (duration: 103.286288ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:45.180717Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39864","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:50:45.203475Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39874","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:50:45.236798Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39894","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:50:45.250766Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39906","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T16:50:45.434176Z","caller":"traceutil/trace.go:172","msg":"trace[1814051823] transaction","detail":"{read_only:false; response_revision:1434; number_of_response:1; }","duration":"135.90318ms","start":"2026-03-30T16:50:45.298255Z","end":"2026-03-30T16:50:45.434158Z","steps":["trace[1814051823] 'process raft request' (duration: 135.775678ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:50:45.494615Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39916","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T16:50:47.942239Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"148.380065ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T16:50:48.043184Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"30.561185ms"} {"level":"warn","ts":"2026-03-30T16:50:48.043261Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"30.646116ms"} {"level":"warn","ts":"2026-03-30T16:50:58.481168Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"144.412404ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:50:58.481296Z","caller":"traceutil/trace.go:172","msg":"trace[1455651765] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1538; }","duration":"144.557045ms","start":"2026-03-30T16:50:58.336720Z","end":"2026-03-30T16:50:58.481277Z","steps":["trace[1455651765] 'range keys from in-memory index tree' (duration: 143.527714ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:51:00.677545Z","caller":"traceutil/trace.go:172","msg":"trace[316182266] transaction","detail":"{read_only:false; response_revision:1546; number_of_response:1; }","duration":"157.100799ms","start":"2026-03-30T16:51:00.520427Z","end":"2026-03-30T16:51:00.677527Z","steps":["trace[316182266] 'process raft request' (duration: 156.943937ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:51:01.933426Z","caller":"traceutil/trace.go:172","msg":"trace[185663601] transaction","detail":"{read_only:false; response_revision:1553; number_of_response:1; }","duration":"117.050943ms","start":"2026-03-30T16:51:01.816351Z","end":"2026-03-30T16:51:01.933402Z","steps":["trace[185663601] 'process raft request' (duration: 116.857322ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:02.511285Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"317.128477ms"} {"level":"warn","ts":"2026-03-30T16:51:02.511349Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"317.196908ms"} {"level":"warn","ts":"2026-03-30T16:51:02.752865Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"574.099719ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:51:02.752928Z","caller":"traceutil/trace.go:172","msg":"trace[1204111921] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1553; }","duration":"574.17212ms","start":"2026-03-30T16:51:02.178745Z","end":"2026-03-30T16:51:02.752917Z","steps":["trace[1204111921] 'range keys from in-memory index tree' (duration: 574.026918ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:02.754321Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"242.858313ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T16:51:02.754391Z","caller":"traceutil/trace.go:172","msg":"trace[48969998] linearizableReadLoop","detail":"{readStateIndex:1812; appliedIndex:1811; }","duration":"303.20682ms","start":"2026-03-30T16:51:02.451173Z","end":"2026-03-30T16:51:02.754379Z","steps":["trace[48969998] 'read index received' (duration: 61.446942ms)","trace[48969998] 'applied index is now lower than readState.Index' (duration: 241.758998ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:51:02.754527Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"303.362632ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T16:51:02.754559Z","caller":"traceutil/trace.go:172","msg":"trace[756452909] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1554; }","duration":"303.395072ms","start":"2026-03-30T16:51:02.451156Z","end":"2026-03-30T16:51:02.754551Z","steps":["trace[756452909] 'agreement among raft nodes before linearized reading' (duration: 303.286581ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:02.754587Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:51:02.451129Z","time spent":"303.449773ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T16:51:02.761098Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"29.217258ms"} {"level":"warn","ts":"2026-03-30T16:51:02.761149Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"29.273469ms"} {"level":"warn","ts":"2026-03-30T16:51:03.080029Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"118.851682ms"} {"level":"warn","ts":"2026-03-30T16:51:03.080096Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"118.937215ms"} {"level":"info","ts":"2026-03-30T16:51:03.080290Z","caller":"traceutil/trace.go:172","msg":"trace[1937347391] linearizableReadLoop","detail":"{readStateIndex:1812; appliedIndex:1813; }","duration":"325.855878ms","start":"2026-03-30T16:51:02.754410Z","end":"2026-03-30T16:51:03.080266Z","steps":["trace[1937347391] 'read index received' (duration: 325.849748ms)","trace[1937347391] 'applied index is now lower than readState.Index' (duration: 4.67µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:51:03.080397Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"327.415759ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:51:03.080443Z","caller":"traceutil/trace.go:172","msg":"trace[1373430929] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1555; }","duration":"327.475231ms","start":"2026-03-30T16:51:02.752955Z","end":"2026-03-30T16:51:03.080430Z","steps":["trace[1373430929] 'agreement among raft nodes before linearized reading' (duration: 327.388018ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:51:03.080454Z","caller":"traceutil/trace.go:172","msg":"trace[1631178198] transaction","detail":"{read_only:false; response_revision:1556; number_of_response:1; }","duration":"396.666312ms","start":"2026-03-30T16:51:02.683767Z","end":"2026-03-30T16:51:03.080433Z","steps":["trace[1631178198] 'process raft request' (duration: 396.505527ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:03.080609Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:51:02.683742Z","time spent":"396.803817ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T16:51:03.469386Z","caller":"traceutil/trace.go:172","msg":"trace[1808958338] linearizableReadLoop","detail":"{readStateIndex:1816; appliedIndex:1816; }","duration":"134.743944ms","start":"2026-03-30T16:51:03.334614Z","end":"2026-03-30T16:51:03.469358Z","steps":["trace[1808958338] 'read index received' (duration: 134.736774ms)","trace[1808958338] 'applied index is now lower than readState.Index' (duration: 5.77µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:51:03.656561Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"187.533034ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T16:51:03.656678Z","caller":"traceutil/trace.go:172","msg":"trace[1245360401] linearizableReadLoop","detail":"{readStateIndex:1817; appliedIndex:1816; }","duration":"159.554041ms","start":"2026-03-30T16:51:03.497108Z","end":"2026-03-30T16:51:03.656662Z","steps":["trace[1245360401] 'read index received' (duration: 1.009345ms)","trace[1245360401] 'applied index is now lower than readState.Index' (duration: 158.543506ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:51:03.656756Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"322.131304ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:51:03.656803Z","caller":"traceutil/trace.go:172","msg":"trace[27235463] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1557; }","duration":"322.180325ms","start":"2026-03-30T16:51:03.334608Z","end":"2026-03-30T16:51:03.656789Z","steps":["trace[27235463] 'agreement among raft nodes before linearized reading' (duration: 134.867269ms)","trace[27235463] 'range keys from in-memory index tree' (duration: 187.242025ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:51:03.656828Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:51:03.334589Z","time spent":"322.232757ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T16:51:03.656933Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"159.802909ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattributesclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:51:03.657345Z","caller":"traceutil/trace.go:172","msg":"trace[2127693583] range","detail":"{range_begin:/registry/volumeattributesclasses; range_end:; response_count:0; response_revision:1557; }","duration":"160.223693ms","start":"2026-03-30T16:51:03.497094Z","end":"2026-03-30T16:51:03.657318Z","steps":["trace[2127693583] 'agreement among raft nodes before linearized reading' (duration: 159.734027ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T16:51:05.193685Z","caller":"traceutil/trace.go:172","msg":"trace[1224400834] transaction","detail":"{read_only:false; response_revision:1560; number_of_response:1; }","duration":"106.182256ms","start":"2026-03-30T16:51:05.087481Z","end":"2026-03-30T16:51:05.193663Z","steps":["trace[1224400834] 'process raft request' (duration: 106.00421ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:05.268660Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"140.248402ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/roles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:51:05.268745Z","caller":"traceutil/trace.go:172","msg":"trace[2064752351] range","detail":"{range_begin:/registry/roles; range_end:; response_count:0; response_revision:1560; }","duration":"140.353375ms","start":"2026-03-30T16:51:05.128373Z","end":"2026-03-30T16:51:05.268726Z","steps":["trace[2064752351] 'agreement among raft nodes before linearized reading' (duration: 93.949042ms)","trace[2064752351] 'range keys from in-memory index tree' (duration: 46.272349ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:51:06.100185Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"141.704062ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T16:51:06.100270Z","caller":"traceutil/trace.go:172","msg":"trace[1668905269] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1563; }","duration":"141.804326ms","start":"2026-03-30T16:51:05.958446Z","end":"2026-03-30T16:51:06.100250Z","steps":["trace[1668905269] 'agreement among raft nodes before linearized reading' (duration: 76.338334ms)","trace[1668905269] 'range keys from in-memory index tree' (duration: 65.235874ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:51:23.521454Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"339.128317ms"} {"level":"warn","ts":"2026-03-30T16:51:23.521497Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"339.176868ms"} {"level":"warn","ts":"2026-03-30T16:51:24.140621Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10218277501228004739,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T16:51:24.152010Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c3874f4929b5956e","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"430.185502ms"} {"level":"warn","ts":"2026-03-30T16:51:24.152068Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"3cf904ec39d3e333","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"430.250174ms"} {"level":"info","ts":"2026-03-30T16:51:24.152048Z","caller":"traceutil/trace.go:172","msg":"trace[2070600323] linearizableReadLoop","detail":"{readStateIndex:1893; appliedIndex:1893; }","duration":"630.304175ms","start":"2026-03-30T16:51:23.521723Z","end":"2026-03-30T16:51:24.152027Z","steps":["trace[2070600323] 'read index received' (duration: 630.296395ms)","trace[2070600323] 'applied index is now lower than readState.Index' (duration: 6.54µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T16:51:24.152185Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"630.44858ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:51:24.152215Z","caller":"traceutil/trace.go:172","msg":"trace[779950123] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1622; }","duration":"630.489161ms","start":"2026-03-30T16:51:23.521716Z","end":"2026-03-30T16:51:24.152206Z","steps":["trace[779950123] 'agreement among raft nodes before linearized reading' (duration: 630.416709ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:24.152249Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:51:23.521693Z","time spent":"630.550583ms","remote":"127.0.0.1:57710","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T16:51:24.179483Z","caller":"traceutil/trace.go:172","msg":"trace[173897770] transaction","detail":"{read_only:false; response_revision:1624; number_of_response:1; }","duration":"657.727243ms","start":"2026-03-30T16:51:23.521741Z","end":"2026-03-30T16:51:24.179468Z","steps":["trace[173897770] 'process raft request' (duration: 657.61454ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:24.179613Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:51:23.521716Z","time spent":"657.832526ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T16:51:24.188309Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10218277501228004742,"received-request-id":10218277501228004739} {"level":"warn","ts":"2026-03-30T16:51:24.194728Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"672.778427ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T16:51:24.194793Z","caller":"traceutil/trace.go:172","msg":"trace[1156623030] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1624; }","duration":"672.869769ms","start":"2026-03-30T16:51:23.521910Z","end":"2026-03-30T16:51:24.194780Z","steps":["trace[1156623030] 'agreement among raft nodes before linearized reading' (duration: 666.582039ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:24.194812Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"513.759729ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T16:51:24.194844Z","caller":"traceutil/trace.go:172","msg":"trace[571125628] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1624; }","duration":"513.79448ms","start":"2026-03-30T16:51:23.681042Z","end":"2026-03-30T16:51:24.194836Z","steps":["trace[571125628] 'agreement among raft nodes before linearized reading' (duration: 507.447388ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T16:51:24.194869Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T16:51:23.681020Z","time spent":"513.843812ms","remote":"127.0.0.1:55514","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "}