{"level":"warn","ts":"2026-03-30T21:29:03.036693Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"warn","ts":"2026-03-30T21:29:03.036855Z","caller":"etcdmain/config.go:270","msg":"--snapshot-count is deprecated in 3.6 and will be decommissioned in 3.7."} {"level":"info","ts":"2026-03-30T21:29:03.036868Z","caller":"etcdmain/etcd.go:64","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.204.45.78:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--feature-gates=InitialCorruptCheck=true","--initial-advertise-peer-urls=https://199.204.45.78:2380","--initial-cluster=controller-2=https://199.204.45.78:2380,controller-1=https://199.204.45.250:2380","--initial-cluster-state=existing","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.204.45.78:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.204.45.78:2380","--name=controller-2","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--watch-progress-notify-interval=5s"]} {"level":"info","ts":"2026-03-30T21:29:03.037129Z","caller":"etcdmain/etcd.go:124","msg":"Initialize and start etcd server","data-dir":"/var/lib/etcd","dir-type":"empty"} {"level":"warn","ts":"2026-03-30T21:29:03.037597Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-30T21:29:03.037627Z","caller":"embed/etcd.go:138","msg":"configuring peer listeners","listen-peer-urls":["https://199.204.45.78:2380"]} {"level":"info","ts":"2026-03-30T21:29:03.037649Z","caller":"embed/etcd.go:544","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-30T21:29:03.038336Z","caller":"embed/etcd.go:146","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.78:2379"]} {"level":"info","ts":"2026-03-30T21:29:03.038467Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.6.4","git-sha":"5400cdc","go-version":"go1.23.11","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-2","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.204.45.78:2380"],"listen-peer-urls":["https://199.204.45.78:2380"],"advertise-client-urls":["https://199.204.45.78:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.78:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"experimental-local-address":"","cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://199.204.45.250:2380,controller-2=https://199.204.45.78:2380","initial-cluster-state":"existing","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"feature-gates":"InitialCorruptCheck=true","initial-corrupt-check":false,"corrupt-check-time-interval":"0s","compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","discovery-token":"","discovery-endpoints":"","discovery-dial-timeout":"2s","discovery-request-timeout":"5s","discovery-keepalive-time":"2s","discovery-keepalive-timeout":"6s","discovery-insecure-transport":true,"discovery-insecure-skip-tls-verify":false,"discovery-cert":"","discovery-key":"","discovery-cacert":"","discovery-user":"","downgrade-check-interval":"5s","max-learners":1,"v2-deprecation":"write-only"} {"level":"info","ts":"2026-03-30T21:29:03.039129Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc0004a2a48}"} {"level":"info","ts":"2026-03-30T21:29:03.044992Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-30T21:29:03.045104Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"6.059192ms"} {"level":"info","ts":"2026-03-30T21:29:03.045240Z","caller":"etcdserver/bootstrap.go:220","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-03-30T21:29:03.045284Z","caller":"etcdserver/bootstrap.go:94","msg":"bootstrapping cluster"} {"level":"info","ts":"2026-03-30T21:29:03.056314Z","caller":"netutil/netutil.go:113","msg":"resolved URL Host","url":"https://199.204.45.250:2380","host":"199.204.45.250:2380","resolved-addr":"199.204.45.250:2380"} {"level":"info","ts":"2026-03-30T21:29:03.056378Z","caller":"netutil/netutil.go:113","msg":"resolved URL Host","url":"https://199.204.45.78:2380","host":"199.204.45.78:2380","resolved-addr":"199.204.45.78:2380"} {"level":"info","ts":"2026-03-30T21:29:03.060760Z","caller":"etcdserver/bootstrap.go:101","msg":"bootstrapping storage"} {"level":"info","ts":"2026-03-30T21:29:03.069989Z","caller":"etcdserver/bootstrap.go:109","msg":"bootstrapping raft"} {"level":"info","ts":"2026-03-30T21:29:03.070048Z","caller":"etcdserver/bootstrap.go:499","msg":"starting local member","local-member-id":"45496929f16fe822","cluster-id":"c785b4aaf4bd3b11"} {"level":"info","ts":"2026-03-30T21:29:03.070422Z","caller":"etcdserver/server.go:312","msg":"bootstrap successfully"} {"level":"info","ts":"2026-03-30T21:29:03.070782Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"45496929f16fe822 switched to configuration voters=()"} {"level":"info","ts":"2026-03-30T21:29:03.072748Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"45496929f16fe822 became follower at term 0"} {"level":"info","ts":"2026-03-30T21:29:03.072828Z","logger":"raft","caller":"v3@v3.6.0/raft.go:493","msg":"newRaft 45496929f16fe822 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"warn","ts":"2026-03-30T21:29:03.080686Z","caller":"auth/store.go:1135","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-30T21:29:03.082213Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-30T21:29:03.085459Z","caller":"storage/quota.go:93","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-30T21:29:03.086076Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.086096Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d","remote-peer-urls":["https://199.204.45.250:2380"]} {"level":"info","ts":"2026-03-30T21:29:03.086116Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.086137Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.088607Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.088741Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.093127Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.093247Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.093066Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.093581Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d","remote-peer-urls":["https://199.204.45.250:2380"]} {"level":"info","ts":"2026-03-30T21:29:03.093716Z","caller":"etcdserver/server.go:598","msg":"starting etcd server","local-member-id":"45496929f16fe822","local-server-version":"3.6.4","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-30T21:29:03.093896Z","caller":"etcdserver/server.go:499","msg":"starting initial election tick advance","election-ticks":10} {"level":"info","ts":"2026-03-30T21:29:03.094606Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-30T21:29:03.094720Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-30T21:29:03.094750Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-30T21:29:03.094720Z","caller":"embed/etcd.go:766","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-30T21:29:03.095202Z","caller":"embed/etcd.go:640","msg":"serving peer traffic","address":"199.204.45.78:2380"} {"level":"info","ts":"2026-03-30T21:29:03.095274Z","caller":"embed/etcd.go:890","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-30T21:29:03.095304Z","caller":"embed/etcd.go:611","msg":"cmux::serve","address":"199.204.45.78:2380"} {"level":"info","ts":"2026-03-30T21:29:03.095230Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"45496929f16fe822","initial-advertise-peer-urls":["https://199.204.45.78:2380"],"listen-peer-urls":["https://199.204.45.78:2380"],"advertise-client-urls":["https://199.204.45.78:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.78:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-30T21:29:03.104472Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.104604Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.105987Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"45496929f16fe822","to":"d98d5f2c752cc52d","stream-type":"stream Message"} {"level":"info","ts":"2026-03-30T21:29:03.105972Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"45496929f16fe822","to":"d98d5f2c752cc52d","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-30T21:29:03.106048Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.106032Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.110715Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.111051Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"45496929f16fe822 [term: 0] received a MsgHeartbeat message with higher term from d98d5f2c752cc52d [term: 2]"} {"level":"info","ts":"2026-03-30T21:29:03.111101Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"45496929f16fe822 became follower at term 2"} {"level":"info","ts":"2026-03-30T21:29:03.111125Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: 45496929f16fe822 elected leader d98d5f2c752cc52d at term 2"} {"level":"info","ts":"2026-03-30T21:29:03.116859Z","caller":"rafthttp/http.go:258","msg":"receiving database snapshot","local-member-id":"45496929f16fe822","remote-snapshot-sender-id":"d98d5f2c752cc52d","incoming-snapshot-index":475,"incoming-snapshot-message-size-bytes":8197,"incoming-snapshot-message-size":"8.2 kB"} {"level":"info","ts":"2026-03-30T21:29:03.141176Z","caller":"snap/db.go:65","msg":"saved database snapshot to disk","path":"/var/lib/etcd/member/snap/00000000000001db.snap.db","bytes":774144,"size":"774 kB"} {"level":"info","ts":"2026-03-30T21:29:03.141260Z","caller":"rafthttp/http.go:287","msg":"received and saved database snapshot","local-member-id":"45496929f16fe822","remote-snapshot-sender-id":"d98d5f2c752cc52d","incoming-snapshot-index":475,"incoming-snapshot-size-bytes":774144,"incoming-snapshot-size":"774 kB","download-took":"24.684038ms"} {"level":"info","ts":"2026-03-30T21:29:03.141485Z","logger":"raft","caller":"v3@v3.6.0/log.go:465","msg":"log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 475, term: 2]"} {"level":"info","ts":"2026-03-30T21:29:03.141630Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"45496929f16fe822 switched to configuration voters=(15676290522495632685) learners=(4992637290782124066)"} {"level":"info","ts":"2026-03-30T21:29:03.141677Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1935","msg":"45496929f16fe822 [commit: 475, lastindex: 475, lastterm: 2] restored snapshot [index: 475, term: 2]"} {"level":"info","ts":"2026-03-30T21:29:03.141700Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1844","msg":"45496929f16fe822 [commit: 475] restored snapshot [index: 475, term: 2]"} {"level":"info","ts":"2026-03-30T21:29:03.141900Z","caller":"etcdserver/server.go:1007","msg":"applying snapshot","current-snapshot-index":0,"current-applied-index":0,"incoming-leader-snapshot-index":475,"incoming-leader-snapshot-term":2} {"level":"info","ts":"2026-03-30T21:29:03.152039Z","caller":"etcdserver/raft.go:276","msg":"applied incoming Raft snapshot","snapshot-index":475} {"level":"info","ts":"2026-03-30T21:29:03.152490Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc0004a2a20}"} {"level":"info","ts":"2026-03-30T21:29:03.153098Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-30T21:29:03.153149Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"770.985µs"} {"level":"info","ts":"2026-03-30T21:29:03.153221Z","caller":"etcdserver/server.go:1055","msg":"restoring lease store"} {"level":"info","ts":"2026-03-30T21:29:03.154288Z","caller":"etcdserver/server.go:1059","msg":"restored lease store"} {"level":"info","ts":"2026-03-30T21:29:03.154308Z","caller":"etcdserver/server.go:1062","msg":"restoring mvcc store"} {"level":"info","ts":"2026-03-30T21:29:03.156776Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":448} {"level":"info","ts":"2026-03-30T21:29:03.156833Z","caller":"etcdserver/server.go:1070","msg":"restored mvcc store","consistent-index":475} {"level":"info","ts":"2026-03-30T21:29:03.156871Z","caller":"etcdserver/server.go:1090","msg":"restoring alarm store"} {"level":"info","ts":"2026-03-30T21:29:03.156983Z","caller":"etcdserver/server.go:1078","msg":"closing old backend file"} {"level":"info","ts":"2026-03-30T21:29:03.158166Z","caller":"etcdserver/server.go:1096","msg":"restored alarm store"} {"level":"info","ts":"2026-03-30T21:29:03.158210Z","caller":"etcdserver/server.go:1099","msg":"restoring auth store"} {"level":"info","ts":"2026-03-30T21:29:03.158246Z","caller":"etcdserver/server.go:1103","msg":"restored auth store"} {"level":"info","ts":"2026-03-30T21:29:03.158265Z","caller":"etcdserver/server.go:1106","msg":"restoring v2 store"} {"level":"info","ts":"2026-03-30T21:29:03.158328Z","caller":"etcdserver/server.go:1080","msg":"closed old backend file"} {"level":"info","ts":"2026-03-30T21:29:03.159081Z","caller":"etcdserver/server.go:1115","msg":"restored v2 store"} {"level":"info","ts":"2026-03-30T21:29:03.159136Z","caller":"etcdserver/server.go:1119","msg":"restoring cluster configuration"} {"level":"info","ts":"2026-03-30T21:29:03.159301Z","caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.6"} {"level":"info","ts":"2026-03-30T21:29:03.159332Z","caller":"membership/cluster.go:297","msg":"recovered/added member from store","cluster-id":"c785b4aaf4bd3b11","local-member-id":"45496929f16fe822","recovered-remote-peer-id":"45496929f16fe822","recovered-remote-peer-urls":["https://199.204.45.78:2380"],"recovered-remote-peer-is-learner":true} {"level":"info","ts":"2026-03-30T21:29:03.159357Z","caller":"membership/cluster.go:297","msg":"recovered/added member from store","cluster-id":"c785b4aaf4bd3b11","local-member-id":"45496929f16fe822","recovered-remote-peer-id":"d98d5f2c752cc52d","recovered-remote-peer-urls":["https://199.204.45.250:2380"],"recovered-remote-peer-is-learner":false} {"level":"info","ts":"2026-03-30T21:29:03.159375Z","caller":"membership/cluster.go:307","msg":"set cluster version from store","cluster-version":"3.6"} {"level":"info","ts":"2026-03-30T21:29:03.159397Z","caller":"etcdserver/server.go:1123","msg":"restored cluster configuration"} {"level":"info","ts":"2026-03-30T21:29:03.159419Z","caller":"etcdserver/server.go:1124","msg":"removing old peers from network"} {"level":"info","ts":"2026-03-30T21:29:03.159443Z","caller":"rafthttp/peer.go:316","msg":"stopping remote peer","remote-peer-id":"d98d5f2c752cc52d"} {"level":"warn","ts":"2026-03-30T21:29:03.159495Z","caller":"rafthttp/stream.go:285","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.159534Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"d98d5f2c752cc52d"} {"level":"warn","ts":"2026-03-30T21:29:03.159584Z","caller":"rafthttp/stream.go:285","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.159602Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.159711Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"warn","ts":"2026-03-30T21:29:03.159863Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d","error":"context canceled"} {"level":"warn","ts":"2026-03-30T21:29:03.159920Z","caller":"rafthttp/peer_status.go:66","msg":"peer became inactive (message send to peer failed)","peer-id":"d98d5f2c752cc52d","error":"failed to read d98d5f2c752cc52d on stream MsgApp v2 (context canceled)"} {"level":"info","ts":"2026-03-30T21:29:03.159949Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"warn","ts":"2026-03-30T21:29:03.160037Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d","error":"context canceled"} {"level":"info","ts":"2026-03-30T21:29:03.160056Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.160074Z","caller":"rafthttp/peer.go:321","msg":"stopped remote peer","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.160097Z","caller":"rafthttp/transport.go:354","msg":"removed remote peer","local-member-id":"45496929f16fe822","removed-remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.160117Z","caller":"etcdserver/server.go:1129","msg":"removed old peers from network"} {"level":"info","ts":"2026-03-30T21:29:03.160129Z","caller":"etcdserver/server.go:1130","msg":"adding peers from new cluster configuration"} {"level":"info","ts":"2026-03-30T21:29:03.160150Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.160170Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.161931Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.162364Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.162441Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.162475Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d","remote-peer-urls":["https://199.204.45.250:2380"]} {"level":"info","ts":"2026-03-30T21:29:03.162492Z","caller":"etcdserver/server.go:1139","msg":"added peers from new cluster configuration"} {"level":"info","ts":"2026-03-30T21:29:03.162513Z","caller":"etcdserver/server.go:1015","msg":"applied snapshot","current-snapshot-index":475,"current-applied-index":475,"incoming-leader-snapshot-index":475,"incoming-leader-snapshot-term":2} {"level":"info","ts":"2026-03-30T21:29:03.162575Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.162745Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.176447Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.176493Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.179952Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.203235Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"45496929f16fe822","to":"d98d5f2c752cc52d","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-30T21:29:03.203330Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.204652Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"45496929f16fe822","to":"d98d5f2c752cc52d","stream-type":"stream Message"} {"level":"info","ts":"2026-03-30T21:29:03.204698Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"d98d5f2c752cc52d"} {"level":"info","ts":"2026-03-30T21:29:03.315154Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"45496929f16fe822 switched to configuration voters=(4992637290782124066 15676290522495632685)"} {"level":"info","ts":"2026-03-30T21:29:03.315665Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"c785b4aaf4bd3b11","local-member-id":"45496929f16fe822","promoted-member-id":"45496929f16fe822"} {"level":"warn","ts":"2026-03-30T21:29:09.749480Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"178.524203ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-30T21:29:10.094762Z","caller":"etcdserver/server.go:1814","msg":"failed to publish local member to cluster through raft","local-member-id":"45496929f16fe822","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.78:2379]}","publish-timeout":"7s","error":"etcdserver: request timed out, possibly due to connection lost"} {"level":"info","ts":"2026-03-30T21:29:10.648794Z","caller":"etcdserver/server.go:1804","msg":"published local member to cluster through raft","local-member-id":"45496929f16fe822","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.78:2379]}","cluster-id":"c785b4aaf4bd3b11","publish-timeout":"7s"} {"level":"info","ts":"2026-03-30T21:29:10.649097Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-30T21:29:10.649387Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-30T21:29:10.649198Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-30T21:29:10.649551Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-03-30T21:29:10.650760Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-30T21:29:10.655224Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.204.45.78:2379"} {"level":"warn","ts":"2026-03-30T21:29:10.655955Z","caller":"v3rpc/grpc.go:52","msg":"etcdserver: failed to register grpc metrics","error":"duplicate metrics collector registration attempted"} {"level":"info","ts":"2026-03-30T21:29:10.656210Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"warn","ts":"2026-03-30T21:29:10.659408Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.78:37680","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:29:10.660440Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"warn","ts":"2026-03-30T21:29:10.660747Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.78:37700","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.661901Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.78:37704","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.662993Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.78:37684","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.663175Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41142","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.663449Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.78:37690","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.663552Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41166","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.663577Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.78:37694","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.663765Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.78:37664","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.665262Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41138","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.666734Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.78:37712","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.673805Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41212","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.686509Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41240","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.711864Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41266","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.719050Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41294","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.730207Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41306","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.738863Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41324","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.762808Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41354","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.772035Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41364","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.781426Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41378","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.800147Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41396","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.808400Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41420","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.816782Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41444","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.826225Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41458","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.836719Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41476","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.844850Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41500","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.853944Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41506","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.863244Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41530","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.901563Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41546","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.910305Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41574","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.921417Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41604","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.931607Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41626","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.941719Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41652","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.953690Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41654","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.962890Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41668","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.972513Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41696","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.983528Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41720","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:10.994594Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41728","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.014611Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41758","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.027292Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41774","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.036371Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41790","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.045118Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41802","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.053344Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41808","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.061112Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41828","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.071757Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41856","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.094111Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41870","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.104203Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41886","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.112365Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41896","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.124508Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41914","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.129966Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41928","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.138982Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41942","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.150905Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41954","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.161634Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41982","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.177616Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41992","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.191871Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42018","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.202084Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42030","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.213114Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42038","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.223391Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42050","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.242050Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42068","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.253881Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42110","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.265186Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42126","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.275020Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42146","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.308103Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42182","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.317923Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42198","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.324794Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42218","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.334741Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42240","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:11.402200Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42248","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:29:20.918128Z","caller":"traceutil/trace.go:172","msg":"trace[773932277] linearizableReadLoop","detail":"{readStateIndex:530; appliedIndex:530; }","duration":"129.185542ms","start":"2026-03-30T21:29:20.788918Z","end":"2026-03-30T21:29:20.918104Z","steps":["trace[773932277] 'read index received' (duration: 129.179012ms)","trace[773932277] 'applied index is now lower than readState.Index' (duration: 5.34µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:20.918307Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"129.325865ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csinodes/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:29:20.918400Z","caller":"traceutil/trace.go:172","msg":"trace[1896314441] range","detail":"{range_begin:/registry/csinodes/controller-2; range_end:; response_count:0; response_revision:496; }","duration":"129.462338ms","start":"2026-03-30T21:29:20.788912Z","end":"2026-03-30T21:29:20.918375Z","steps":["trace[1896314441] 'agreement among raft nodes before linearized reading' (duration: 129.288464ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:26.565033Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003359842,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:29:27.066069Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003359842,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:29:27.100055Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:26.064089Z","time spent":"1.035962602s","remote":"127.0.0.1:41262","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-30T21:29:27.114736Z","caller":"traceutil/trace.go:172","msg":"trace[671790499] linearizableReadLoop","detail":"{readStateIndex:561; appliedIndex:562; }","duration":"1.050139135s","start":"2026-03-30T21:29:26.064573Z","end":"2026-03-30T21:29:27.114712Z","steps":["trace[671790499] 'read index received' (duration: 1.050132775s)","trace[671790499] 'applied index is now lower than readState.Index' (duration: 5.75µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:27.114922Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.050322419s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/kube-system/kube-proxy\" limit:1 ","response":"range_response_count:1 size:185"} {"level":"info","ts":"2026-03-30T21:29:27.115029Z","caller":"traceutil/trace.go:172","msg":"trace[505911109] range","detail":"{range_begin:/registry/serviceaccounts/kube-system/kube-proxy; range_end:; response_count:1; response_revision:525; }","duration":"1.050443402s","start":"2026-03-30T21:29:26.064567Z","end":"2026-03-30T21:29:27.115010Z","steps":["trace[505911109] 'agreement among raft nodes before linearized reading' (duration: 1.050215307s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:27.115080Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:26.064539Z","time spent":"1.050527143s","remote":"127.0.0.1:41518","response type":"/etcdserverpb.KV/Range","request count":0,"request size":52,"response count":1,"response size":208,"request content":"key:\"/registry/serviceaccounts/kube-system/kube-proxy\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:29:27.385053Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003359846,"received-request-id":16727104867003359842} {"level":"info","ts":"2026-03-30T21:29:27.386729Z","caller":"traceutil/trace.go:172","msg":"trace[856173914] transaction","detail":"{read_only:false; response_revision:529; number_of_response:1; }","duration":"285.042329ms","start":"2026-03-30T21:29:27.101652Z","end":"2026-03-30T21:29:27.386694Z","steps":["trace[856173914] 'process raft request' (duration: 284.957927ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:29:27.387109Z","caller":"traceutil/trace.go:172","msg":"trace[802711511] linearizableReadLoop","detail":"{readStateIndex:565; appliedIndex:567; }","duration":"272.271948ms","start":"2026-03-30T21:29:27.114793Z","end":"2026-03-30T21:29:27.387065Z","steps":["trace[802711511] 'read index received' (duration: 272.265198ms)","trace[802711511] 'applied index is now lower than readState.Index' (duration: 5.59µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:27.387365Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"538.212324ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:29:27.387413Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"922.552593ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:29:27.387445Z","caller":"traceutil/trace.go:172","msg":"trace[1248726481] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:529; }","duration":"538.299366ms","start":"2026-03-30T21:29:26.849132Z","end":"2026-03-30T21:29:27.387432Z","steps":["trace[1248726481] 'agreement among raft nodes before linearized reading' (duration: 538.174503ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:27.387482Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:26.849112Z","time spent":"538.361057ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:29:27.387816Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.151025006s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T21:29:27.387478Z","caller":"traceutil/trace.go:172","msg":"trace[432560465] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:529; }","duration":"922.608604ms","start":"2026-03-30T21:29:26.464838Z","end":"2026-03-30T21:29:27.387447Z","steps":["trace[432560465] 'agreement among raft nodes before linearized reading' (duration: 922.530652ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:27.387963Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"800.450532ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 ","response":"range_response_count:1 size:6085"} {"level":"info","ts":"2026-03-30T21:29:27.388032Z","caller":"traceutil/trace.go:172","msg":"trace[1699814155] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-2; range_end:; response_count:1; response_revision:529; }","duration":"800.523503ms","start":"2026-03-30T21:29:26.587494Z","end":"2026-03-30T21:29:27.388018Z","steps":["trace[1699814155] 'agreement among raft nodes before linearized reading' (duration: 800.058283ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:27.388115Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:26.587466Z","time spent":"800.638246ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":1,"response size":6108,"request content":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 "} {"level":"info","ts":"2026-03-30T21:29:27.388169Z","caller":"traceutil/trace.go:172","msg":"trace[944474706] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:529; }","duration":"1.151075367s","start":"2026-03-30T21:29:26.236776Z","end":"2026-03-30T21:29:27.387852Z","steps":["trace[944474706] 'agreement among raft nodes before linearized reading' (duration: 1.150981655s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:27.388416Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:26.236752Z","time spent":"1.151593528s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:29:27.390104Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"273.679888ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-proxy-fxqfv\" limit:1 ","response":"range_response_count:1 size:3429"} {"level":"info","ts":"2026-03-30T21:29:27.390249Z","caller":"traceutil/trace.go:172","msg":"trace[393927176] range","detail":"{range_begin:/registry/pods/kube-system/kube-proxy-fxqfv; range_end:; response_count:1; response_revision:529; }","duration":"273.832692ms","start":"2026-03-30T21:29:27.116391Z","end":"2026-03-30T21:29:27.390224Z","steps":["trace[393927176] 'agreement among raft nodes before linearized reading' (duration: 273.553105ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:29:27.491155Z","caller":"traceutil/trace.go:172","msg":"trace[1758818040] linearizableReadLoop","detail":"{readStateIndex:567; appliedIndex:567; }","duration":"101.259737ms","start":"2026-03-30T21:29:27.389868Z","end":"2026-03-30T21:29:27.491128Z","steps":["trace[1758818040] 'read index received' (duration: 101.254397ms)","trace[1758818040] 'applied index is now lower than readState.Index' (duration: 4.59µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:27.634465Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"244.976014ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:29:27.634646Z","caller":"traceutil/trace.go:172","msg":"trace[1473970543] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:529; }","duration":"245.174219ms","start":"2026-03-30T21:29:27.389450Z","end":"2026-03-30T21:29:27.634624Z","steps":["trace[1473970543] 'agreement among raft nodes before linearized reading' (duration: 101.853031ms)","trace[1473970543] 'range keys from in-memory index tree' (duration: 143.094323ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:27.634988Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"142.737215ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:29:27.665818Z","caller":"traceutil/trace.go:172","msg":"trace[341823621] transaction","detail":"{read_only:false; response_revision:531; number_of_response:1; }","duration":"256.044108ms","start":"2026-03-30T21:29:27.409753Z","end":"2026-03-30T21:29:27.665797Z","steps":["trace[341823621] 'process raft request' (duration: 255.934826ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:29:27.666958Z","caller":"traceutil/trace.go:172","msg":"trace[433180358] linearizableReadLoop","detail":"{readStateIndex:568; appliedIndex:569; }","duration":"175.60278ms","start":"2026-03-30T21:29:27.491344Z","end":"2026-03-30T21:29:27.666947Z","steps":["trace[433180358] 'read index received' (duration: 175.59753ms)","trace[433180358] 'applied index is now lower than readState.Index' (duration: 4.56µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:27.667053Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"201.425901ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:29:27.667153Z","caller":"traceutil/trace.go:172","msg":"trace[603097131] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:531; }","duration":"201.542414ms","start":"2026-03-30T21:29:27.465588Z","end":"2026-03-30T21:29:27.667131Z","steps":["trace[603097131] 'agreement among raft nodes before linearized reading' (duration: 201.38433ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:27.667157Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"273.213848ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-proxy-fxqfv.18a1bbc111baf76d\" limit:1 ","response":"range_response_count:1 size:776"} {"level":"info","ts":"2026-03-30T21:29:27.667298Z","caller":"traceutil/trace.go:172","msg":"trace[1503184796] range","detail":"{range_begin:/registry/events/kube-system/kube-proxy-fxqfv.18a1bbc111baf76d; range_end:; response_count:1; response_revision:531; }","duration":"273.356582ms","start":"2026-03-30T21:29:27.393923Z","end":"2026-03-30T21:29:27.667280Z","steps":["trace[1503184796] 'agreement among raft nodes before linearized reading' (duration: 273.064204ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:29:27.955914Z","caller":"traceutil/trace.go:172","msg":"trace[775798112] transaction","detail":"{read_only:false; response_revision:532; number_of_response:1; }","duration":"282.932563ms","start":"2026-03-30T21:29:27.672955Z","end":"2026-03-30T21:29:27.955888Z","steps":["trace[775798112] 'process raft request' (duration: 282.500703ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:29:28.169886Z","caller":"traceutil/trace.go:172","msg":"trace[1587097753] linearizableReadLoop","detail":"{readStateIndex:570; appliedIndex:571; }","duration":"319.271836ms","start":"2026-03-30T21:29:27.850583Z","end":"2026-03-30T21:29:28.169855Z","steps":["trace[1587097753] 'read index received' (duration: 319.260156ms)","trace[1587097753] 'applied index is now lower than readState.Index' (duration: 10.36µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:28.411629Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"561.024835ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:29:28.411772Z","caller":"traceutil/trace.go:172","msg":"trace[1332860834] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:533; }","duration":"561.170358ms","start":"2026-03-30T21:29:27.850567Z","end":"2026-03-30T21:29:28.411737Z","steps":["trace[1332860834] 'agreement among raft nodes before linearized reading' (duration: 319.387069ms)","trace[1332860834] 'range keys from in-memory index tree' (duration: 241.586985ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:28.411830Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:27.850538Z","time spent":"561.27956ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:29:28.411896Z","caller":"traceutil/trace.go:172","msg":"trace[1335791794] linearizableReadLoop","detail":"{readStateIndex:573; appliedIndex:571; }","duration":"241.905352ms","start":"2026-03-30T21:29:28.169963Z","end":"2026-03-30T21:29:28.411868Z","steps":["trace[1335791794] 'read index received' (duration: 239.003418ms)","trace[1335791794] 'applied index is now lower than readState.Index' (duration: 2.900924ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:28.412067Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"488.318929ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:29:28.412046Z","caller":"traceutil/trace.go:172","msg":"trace[925423075] transaction","detail":"{read_only:false; response_revision:534; number_of_response:1; }","duration":"674.443211ms","start":"2026-03-30T21:29:27.737575Z","end":"2026-03-30T21:29:28.412018Z","steps":["trace[925423075] 'process raft request' (duration: 669.306918ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:29:28.412111Z","caller":"traceutil/trace.go:172","msg":"trace[741948938] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:535; }","duration":"488.39419ms","start":"2026-03-30T21:29:27.923707Z","end":"2026-03-30T21:29:28.412101Z","steps":["trace[741948938] 'agreement among raft nodes before linearized reading' (duration: 488.282118ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:28.412166Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:27.923649Z","time spent":"488.507882ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:29:28.412657Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:27.737551Z","time spent":"674.584415ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7571,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:29:28.783443Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"115.622853ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:29:28.783537Z","caller":"traceutil/trace.go:172","msg":"trace[1703367562] linearizableReadLoop","detail":"{readStateIndex:574; appliedIndex:573; }","duration":"317.773753ms","start":"2026-03-30T21:29:28.465750Z","end":"2026-03-30T21:29:28.783523Z","steps":["trace[1703367562] 'read index received' (duration: 203.158623ms)","trace[1703367562] 'applied index is now lower than readState.Index' (duration: 114.61426ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:29:28.783615Z","caller":"traceutil/trace.go:172","msg":"trace[2025899775] transaction","detail":"{read_only:false; response_revision:536; number_of_response:1; }","duration":"352.914933ms","start":"2026-03-30T21:29:28.430690Z","end":"2026-03-30T21:29:28.783605Z","steps":["trace[2025899775] 'process raft request' (duration: 237.064226ms)","trace[2025899775] 'compare' (duration: 115.514359ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:28.783665Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:28.430646Z","time spent":"352.992014ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5024,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:29:28.783749Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"317.999698ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:29:28.783784Z","caller":"traceutil/trace.go:172","msg":"trace[233908170] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:536; }","duration":"318.036888ms","start":"2026-03-30T21:29:28.465739Z","end":"2026-03-30T21:29:28.783776Z","steps":["trace[233908170] 'agreement among raft nodes before linearized reading' (duration: 317.980287ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:28.785832Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"141.542008ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-30T21:29:28.785876Z","caller":"traceutil/trace.go:172","msg":"trace[983861238] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:536; }","duration":"141.588659ms","start":"2026-03-30T21:29:28.644278Z","end":"2026-03-30T21:29:28.785867Z","steps":["trace[983861238] 'agreement among raft nodes before linearized reading' (duration: 141.455196ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:29:29.108442Z","caller":"traceutil/trace.go:172","msg":"trace[687569898] transaction","detail":"{read_only:false; response_revision:537; number_of_response:1; }","duration":"315.735785ms","start":"2026-03-30T21:29:28.792666Z","end":"2026-03-30T21:29:29.108402Z","steps":["trace[687569898] 'process raft request' (duration: 281.135566ms)","trace[687569898] 'compare' (duration: 34.355503ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:29.108631Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:29:28.792653Z","time spent":"315.877738ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4802,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:29:34.565561Z","caller":"traceutil/trace.go:172","msg":"trace[1562700233] linearizableReadLoop","detail":"{readStateIndex:604; appliedIndex:604; }","duration":"100.352901ms","start":"2026-03-30T21:29:34.465185Z","end":"2026-03-30T21:29:34.565538Z","steps":["trace[1562700233] 'read index received' (duration: 100.335201ms)","trace[1562700233] 'applied index is now lower than readState.Index' (duration: 16.89µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:34.565668Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"100.474184ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:29:34.565691Z","caller":"traceutil/trace.go:172","msg":"trace[1096283300] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:563; }","duration":"100.508454ms","start":"2026-03-30T21:29:34.465175Z","end":"2026-03-30T21:29:34.565684Z","steps":["trace[1096283300] 'agreement among raft nodes before linearized reading' (duration: 100.448983ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:47.832423Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:48948","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:29:48.287489Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"45496929f16fe822 switched to configuration voters=(4992637290782124066 15676290522495632685) learners=(8336499844540403861)"} {"level":"info","ts":"2026-03-30T21:29:48.287753Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"c785b4aaf4bd3b11","local-member-id":"45496929f16fe822","added-peer-id":"73b132709fee1095","added-peer-peer-urls":["https://199.204.45.72:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-30T21:29:48.287801Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:48.287842Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:48.288636Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:48.291949Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:48.294916Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:48.294964Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:48.294982Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:48.294996Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095","remote-peer-urls":["https://199.204.45.72:2380"]} {"level":"info","ts":"2026-03-30T21:29:48.295215Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"45496929f16fe822","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"73b132709fee1095"} {"level":"warn","ts":"2026-03-30T21:29:48.328076Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:49060","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:48.350348Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:49108","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:48.855176Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:49126","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:49.852624Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:49144","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:50.852545Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:49192","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:29:51.503978Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"45496929f16fe822","to":"73b132709fee1095","stream-type":"stream Message"} {"level":"info","ts":"2026-03-30T21:29:51.504057Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.504112Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.511803Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"45496929f16fe822","to":"73b132709fee1095","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-30T21:29:51.511859Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.519399Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.521348Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"warn","ts":"2026-03-30T21:29:51.563627Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:29:51.563627Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095","error":"EOF"} {"level":"info","ts":"2026-03-30T21:29:51.577485Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"45496929f16fe822","to":"73b132709fee1095","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-30T21:29:51.577513Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.577526Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.581907Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"45496929f16fe822","to":"73b132709fee1095","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-30T21:29:51.581962Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.581982Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.606471Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.616309Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"45496929f16fe822","remote-peer-id":"73b132709fee1095"} {"level":"info","ts":"2026-03-30T21:29:51.886568Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"45496929f16fe822 switched to configuration voters=(4992637290782124066 8336499844540403861 15676290522495632685)"} {"level":"info","ts":"2026-03-30T21:29:51.886762Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"c785b4aaf4bd3b11","local-member-id":"45496929f16fe822","promoted-member-id":"73b132709fee1095"} {"level":"warn","ts":"2026-03-30T21:29:51.937821Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:49280","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:29:58.575948Z","caller":"traceutil/trace.go:172","msg":"trace[484988199] linearizableReadLoop","detail":"{readStateIndex:741; appliedIndex:741; }","duration":"150.2793ms","start":"2026-03-30T21:29:58.425644Z","end":"2026-03-30T21:29:58.575923Z","steps":["trace[484988199] 'read index received' (duration: 150.27402ms)","trace[484988199] 'applied index is now lower than readState.Index' (duration: 4.23µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:29:58.576210Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"150.531157ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-30T21:29:58.576294Z","caller":"traceutil/trace.go:172","msg":"trace[58392221] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:686; }","duration":"150.65353ms","start":"2026-03-30T21:29:58.425626Z","end":"2026-03-30T21:29:58.576280Z","steps":["trace[58392221] 'agreement among raft nodes before linearized reading' (duration: 150.395144ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:29:58.577986Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"113.119481ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:29:58.578088Z","caller":"traceutil/trace.go:172","msg":"trace[609455707] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:686; }","duration":"113.226444ms","start":"2026-03-30T21:29:58.464847Z","end":"2026-03-30T21:29:58.578073Z","steps":["trace[609455707] 'agreement among raft nodes before linearized reading' (duration: 113.045679ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:02.672235Z","caller":"traceutil/trace.go:172","msg":"trace[376760771] transaction","detail":"{read_only:false; response_revision:707; number_of_response:1; }","duration":"136.641949ms","start":"2026-03-30T21:30:02.535562Z","end":"2026-03-30T21:30:02.672204Z","steps":["trace[376760771] 'process raft request' (duration: 136.408543ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:03.294043Z","caller":"traceutil/trace.go:172","msg":"trace[1947074309] transaction","detail":"{read_only:false; response_revision:710; number_of_response:1; }","duration":"104.459365ms","start":"2026-03-30T21:30:03.189561Z","end":"2026-03-30T21:30:03.294020Z","steps":["trace[1947074309] 'process raft request' (duration: 104.2283ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:03.438322Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"113.151204ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.78\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-30T21:30:03.438672Z","caller":"traceutil/trace.go:172","msg":"trace[607425695] range","detail":"{range_begin:/registry/masterleases/199.204.45.78; range_end:; response_count:1; response_revision:710; }","duration":"113.518042ms","start":"2026-03-30T21:30:03.325137Z","end":"2026-03-30T21:30:03.438655Z","steps":["trace[607425695] 'range keys from in-memory index tree' (duration: 111.338169ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:03.661705Z","caller":"traceutil/trace.go:172","msg":"trace[1548669398] linearizableReadLoop","detail":"{readStateIndex:769; appliedIndex:769; }","duration":"195.938491ms","start":"2026-03-30T21:30:03.465738Z","end":"2026-03-30T21:30:03.661676Z","steps":["trace[1548669398] 'read index received' (duration: 195.93288ms)","trace[1548669398] 'applied index is now lower than readState.Index' (duration: 4.311µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:03.680053Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"214.295559ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:03.680104Z","caller":"traceutil/trace.go:172","msg":"trace[1819663911] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:711; }","duration":"214.35979ms","start":"2026-03-30T21:30:03.465729Z","end":"2026-03-30T21:30:03.680089Z","steps":["trace[1819663911] 'agreement among raft nodes before linearized reading' (duration: 196.041623ms)","trace[1819663911] 'range keys from in-memory index tree' (duration: 18.234036ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:30:11.380954Z","caller":"traceutil/trace.go:172","msg":"trace[1009590543] linearizableReadLoop","detail":"{readStateIndex:805; appliedIndex:806; }","duration":"170.797206ms","start":"2026-03-30T21:30:11.210136Z","end":"2026-03-30T21:30:11.380933Z","steps":["trace[1009590543] 'read index received' (duration: 170.791606ms)","trace[1009590543] 'applied index is now lower than readState.Index' (duration: 4.33µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:11.381136Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"170.978721ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/networkpolicies\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:11.381204Z","caller":"traceutil/trace.go:172","msg":"trace[618580453] range","detail":"{range_begin:/registry/networkpolicies; range_end:; response_count:0; response_revision:742; }","duration":"171.058683ms","start":"2026-03-30T21:30:11.210130Z","end":"2026-03-30T21:30:11.381188Z","steps":["trace[618580453] 'agreement among raft nodes before linearized reading' (duration: 170.94155ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:14.533269Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360091,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:30:14.620043Z","caller":"traceutil/trace.go:172","msg":"trace[652969078] linearizableReadLoop","detail":"{readStateIndex:816; appliedIndex:816; }","duration":"587.690185ms","start":"2026-03-30T21:30:14.032323Z","end":"2026-03-30T21:30:14.620013Z","steps":["trace[652969078] 'read index received' (duration: 587.680485ms)","trace[652969078] 'applied index is now lower than readState.Index' (duration: 8.04µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:14.620300Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"587.961412ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-30T21:30:14.620388Z","caller":"traceutil/trace.go:172","msg":"trace[1679032138] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:750; }","duration":"588.064234ms","start":"2026-03-30T21:30:14.032304Z","end":"2026-03-30T21:30:14.620368Z","steps":["trace[1679032138] 'agreement among raft nodes before linearized reading' (duration: 587.814459ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:14.620441Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:14.032269Z","time spent":"588.157796ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":463,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:14.672379Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"604.229216ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/certificatesigningrequests\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:30:14.672400Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"594.820807ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csidrivers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:14.672456Z","caller":"traceutil/trace.go:172","msg":"trace[338824387] range","detail":"{range_begin:/registry/certificatesigningrequests; range_end:; response_count:0; response_revision:750; }","duration":"604.327539ms","start":"2026-03-30T21:30:14.068111Z","end":"2026-03-30T21:30:14.672438Z","steps":["trace[338824387] 'agreement among raft nodes before linearized reading' (duration: 604.170005ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:14.672510Z","caller":"traceutil/trace.go:172","msg":"trace[39356156] range","detail":"{range_begin:/registry/csidrivers; range_end:; response_count:0; response_revision:750; }","duration":"594.931481ms","start":"2026-03-30T21:30:14.077550Z","end":"2026-03-30T21:30:14.672482Z","steps":["trace[39356156] 'agreement among raft nodes before linearized reading' (duration: 594.765317ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:14.672515Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:14.068088Z","time spent":"604.414351ms","remote":"127.0.0.1:41632","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":0,"response size":28,"request content":"key:\"/registry/certificatesigningrequests\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:14.672550Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:14.077526Z","time spent":"595.013462ms","remote":"127.0.0.1:41940","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/csidrivers\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:14.672628Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"206.522219ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:14.672870Z","caller":"traceutil/trace.go:172","msg":"trace[1883708359] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:750; }","duration":"206.98541ms","start":"2026-03-30T21:30:14.465860Z","end":"2026-03-30T21:30:14.672845Z","steps":["trace[1883708359] 'agreement among raft nodes before linearized reading' (duration: 206.450628ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:15.402078Z","caller":"traceutil/trace.go:172","msg":"trace[233566082] linearizableReadLoop","detail":"{readStateIndex:819; appliedIndex:820; }","duration":"257.298799ms","start":"2026-03-30T21:30:15.144755Z","end":"2026-03-30T21:30:15.402054Z","steps":["trace[233566082] 'read index received' (duration: 257.292238ms)","trace[233566082] 'applied index is now lower than readState.Index' (duration: 5.141µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:15.402297Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"257.527015ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T21:30:15.402340Z","caller":"traceutil/trace.go:172","msg":"trace[1211525572] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:754; }","duration":"257.584376ms","start":"2026-03-30T21:30:15.144743Z","end":"2026-03-30T21:30:15.402327Z","steps":["trace[1211525572] 'agreement among raft nodes before linearized reading' (duration: 257.414112ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:15.673942Z","caller":"traceutil/trace.go:172","msg":"trace[1736189096] linearizableReadLoop","detail":"{readStateIndex:820; appliedIndex:821; }","duration":"239.941117ms","start":"2026-03-30T21:30:15.433960Z","end":"2026-03-30T21:30:15.673901Z","steps":["trace[1736189096] 'read index received' (duration: 239.934387ms)","trace[1736189096] 'applied index is now lower than readState.Index' (duration: 5.71µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:15.864542Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"430.574336ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:15.864622Z","caller":"traceutil/trace.go:172","msg":"trace[778725927] range","detail":"{range_begin:/registry/controllers; range_end:; response_count:0; response_revision:755; }","duration":"430.662328ms","start":"2026-03-30T21:30:15.433944Z","end":"2026-03-30T21:30:15.864607Z","steps":["trace[778725927] 'agreement among raft nodes before linearized reading' (duration: 240.057269ms)","trace[778725927] 'range keys from in-memory index tree' (duration: 190.493006ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:15.864659Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:15.433922Z","time spent":"430.72761ms","remote":"127.0.0.1:41528","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/controllers\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:15.864824Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"190.724793ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:30:15.888527Z","caller":"traceutil/trace.go:172","msg":"trace[1029463760] linearizableReadLoop","detail":"{readStateIndex:825; appliedIndex:826; }","duration":"214.50006ms","start":"2026-03-30T21:30:15.674007Z","end":"2026-03-30T21:30:15.888507Z","steps":["trace[1029463760] 'read index received' (duration: 214.49543ms)","trace[1029463760] 'applied index is now lower than readState.Index' (duration: 3.52µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:15.888670Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"423.492624ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:15.888724Z","caller":"traceutil/trace.go:172","msg":"trace[1751776132] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:760; }","duration":"423.550156ms","start":"2026-03-30T21:30:15.465150Z","end":"2026-03-30T21:30:15.888700Z","steps":["trace[1751776132] 'agreement among raft nodes before linearized reading' (duration: 423.474534ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:15.890699Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"113.035867ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/roles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:30:15.890684Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"139.482938ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumes\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:15.890734Z","caller":"traceutil/trace.go:172","msg":"trace[1287673598] range","detail":"{range_begin:/registry/roles; range_end:; response_count:0; response_revision:760; }","duration":"113.072467ms","start":"2026-03-30T21:30:15.777653Z","end":"2026-03-30T21:30:15.890725Z","steps":["trace[1287673598] 'agreement among raft nodes before linearized reading' (duration: 112.964955ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:15.890748Z","caller":"traceutil/trace.go:172","msg":"trace[1224347238] range","detail":"{range_begin:/registry/persistentvolumes; range_end:; response_count:0; response_revision:760; }","duration":"139.56124ms","start":"2026-03-30T21:30:15.751174Z","end":"2026-03-30T21:30:15.890735Z","steps":["trace[1224347238] 'agreement among raft nodes before linearized reading' (duration: 139.447627ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:15.891009Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"128.743357ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:30:15.891128Z","caller":"traceutil/trace.go:172","msg":"trace[2065592237] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:760; }","duration":"128.88663ms","start":"2026-03-30T21:30:15.762218Z","end":"2026-03-30T21:30:15.891104Z","steps":["trace[2065592237] 'agreement among raft nodes before linearized reading' (duration: 128.520102ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:37.741280Z","caller":"traceutil/trace.go:172","msg":"trace[1272815899] linearizableReadLoop","detail":"{readStateIndex:979; appliedIndex:979; }","duration":"269.252722ms","start":"2026-03-30T21:30:37.472004Z","end":"2026-03-30T21:30:37.741257Z","steps":["trace[1272815899] 'read index received' (duration: 269.248162ms)","trace[1272815899] 'applied index is now lower than readState.Index' (duration: 3.72µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:37.741379Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"269.366245ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:37.741489Z","caller":"traceutil/trace.go:172","msg":"trace[1411449119] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:897; }","duration":"269.456227ms","start":"2026-03-30T21:30:37.471995Z","end":"2026-03-30T21:30:37.741451Z","steps":["trace[1411449119] 'agreement among raft nodes before linearized reading' (duration: 269.351654ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:38.431812Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360253,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:38.932188Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360253,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:39.433207Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360253,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:39.538026Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.68569267s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T21:30:39.538572Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.07294077s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:39.538656Z","caller":"traceutil/trace.go:172","msg":"trace[1093159453] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:897; }","duration":"1.073024062s","start":"2026-03-30T21:30:38.465615Z","end":"2026-03-30T21:30:39.538639Z","steps":["trace[1093159453] 'range keys from in-memory index tree' (duration: 1.072893979s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:39.538390Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.688206212s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:39.539781Z","caller":"traceutil/trace.go:172","msg":"trace[1824797259] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:897; }","duration":"1.689609317s","start":"2026-03-30T21:30:37.850154Z","end":"2026-03-30T21:30:39.539763Z","steps":["trace[1824797259] 'range keys from in-memory index tree' (duration: 1.68653592s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:39.539824Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:37.850135Z","time spent":"1.689675458s","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:30:39.934414Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360253,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:40.434820Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360253,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:40.936505Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360253,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:30:41.009282Z","caller":"traceutil/trace.go:172","msg":"trace[1744473840] linearizableReadLoop","detail":"{readStateIndex:980; appliedIndex:981; }","duration":"3.078139342s","start":"2026-03-30T21:30:37.931119Z","end":"2026-03-30T21:30:41.009258Z","steps":["trace[1744473840] 'read index received' (duration: 3.078128662s)","trace[1744473840] 'applied index is now lower than readState.Index' (duration: 9.33µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:41.009409Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.078267215s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/daemonsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:41.009481Z","caller":"traceutil/trace.go:172","msg":"trace[15227489] range","detail":"{range_begin:/registry/daemonsets; range_end:; response_count:0; response_revision:900; }","duration":"3.078353107s","start":"2026-03-30T21:30:37.931115Z","end":"2026-03-30T21:30:41.009468Z","steps":["trace[15227489] 'agreement among raft nodes before linearized reading' (duration: 3.078211283s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.009516Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:37.931099Z","time spent":"3.078407309s","remote":"127.0.0.1:42028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/daemonsets\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:41.016918Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360254,"received-request-id":16727104867003360253} {"level":"warn","ts":"2026-03-30T21:30:41.060438Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.127707498s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"warn","ts":"2026-03-30T21:30:41.060473Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.920374683s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T21:30:41.060541Z","caller":"traceutil/trace.go:172","msg":"trace[1758092406] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:902; }","duration":"1.920446825s","start":"2026-03-30T21:30:39.140075Z","end":"2026-03-30T21:30:41.060522Z","steps":["trace[1758092406] 'agreement among raft nodes before linearized reading' (duration: 1.920278541s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:41.060546Z","caller":"traceutil/trace.go:172","msg":"trace[1196779219] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:902; }","duration":"3.12780349s","start":"2026-03-30T21:30:37.932698Z","end":"2026-03-30T21:30:41.060502Z","steps":["trace[1196779219] 'agreement among raft nodes before linearized reading' (duration: 3.127562983s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.060582Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:39.140055Z","time spent":"1.920517877s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":506,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:41.060648Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:37.932679Z","time spent":"3.127908763s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:41.060803Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.023603464s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:41.060840Z","caller":"traceutil/trace.go:172","msg":"trace[1456367256] range","detail":"{range_begin:/registry/horizontalpodautoscalers; range_end:; response_count:0; response_revision:902; }","duration":"2.023638905s","start":"2026-03-30T21:30:39.037190Z","end":"2026-03-30T21:30:41.060829Z","steps":["trace[1456367256] 'agreement among raft nodes before linearized reading' (duration: 2.023568634s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.060844Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.074263695s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-30T21:30:41.060868Z","caller":"traceutil/trace.go:172","msg":"trace[93312534] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:902; }","duration":"2.074285525s","start":"2026-03-30T21:30:38.986575Z","end":"2026-03-30T21:30:41.060860Z","steps":["trace[93312534] 'agreement among raft nodes before linearized reading' (duration: 2.074246704s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.060889Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:38.986553Z","time spent":"2.074330917s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":463,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:41.060811Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"816.352153ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingressclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:30:41.060871Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"636.935012ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/limitranges\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:30:41.060866Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:39.037167Z","time spent":"2.023691396s","remote":"127.0.0.1:41562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":38,"response count":0,"response size":28,"request content":"key:\"/registry/horizontalpodautoscalers\" limit:1 "} {"level":"info","ts":"2026-03-30T21:30:41.060939Z","caller":"traceutil/trace.go:172","msg":"trace[1769673170] range","detail":"{range_begin:/registry/limitranges; range_end:; response_count:0; response_revision:902; }","duration":"637.009674ms","start":"2026-03-30T21:30:40.423917Z","end":"2026-03-30T21:30:41.060926Z","steps":["trace[1769673170] 'agreement among raft nodes before linearized reading' (duration: 636.904941ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.060975Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:40.423890Z","time spent":"637.075376ms","remote":"127.0.0.1:41392","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/limitranges\" limit:1 "} {"level":"info","ts":"2026-03-30T21:30:41.060930Z","caller":"traceutil/trace.go:172","msg":"trace[455689678] range","detail":"{range_begin:/registry/ingressclasses; range_end:; response_count:0; response_revision:902; }","duration":"816.466715ms","start":"2026-03-30T21:30:40.244448Z","end":"2026-03-30T21:30:41.060915Z","steps":["trace[455689678] 'agreement among raft nodes before linearized reading' (duration: 816.330222ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.060985Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.078152386s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T21:30:41.061023Z","caller":"traceutil/trace.go:172","msg":"trace[703543263] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:902; }","duration":"3.078188197s","start":"2026-03-30T21:30:37.982825Z","end":"2026-03-30T21:30:41.061014Z","steps":["trace[703543263] 'agreement among raft nodes before linearized reading' (duration: 3.078095265s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.060906Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.018749508s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterrolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:30:41.061024Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:40.244425Z","time spent":"816.588099ms","remote":"127.0.0.1:41708","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/ingressclasses\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:41.061047Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:37.982804Z","time spent":"3.078236747s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"info","ts":"2026-03-30T21:30:41.061051Z","caller":"traceutil/trace.go:172","msg":"trace[1968447886] range","detail":"{range_begin:/registry/clusterrolebindings; range_end:; response_count:0; response_revision:902; }","duration":"3.018892753s","start":"2026-03-30T21:30:38.042150Z","end":"2026-03-30T21:30:41.061043Z","steps":["trace[1968447886] 'agreement among raft nodes before linearized reading' (duration: 3.018731219s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.061132Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:38.042130Z","time spent":"3.018995115s","remote":"127.0.0.1:41830","response type":"/etcdserverpb.KV/Range","request count":0,"request size":33,"response count":0,"response size":28,"request content":"key:\"/registry/clusterrolebindings\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:41.060803Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.202590577s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterroles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:41.061162Z","caller":"traceutil/trace.go:172","msg":"trace[1703324417] range","detail":"{range_begin:/registry/clusterroles; range_end:; response_count:0; response_revision:902; }","duration":"2.202954806s","start":"2026-03-30T21:30:38.858202Z","end":"2026-03-30T21:30:41.061156Z","steps":["trace[1703324417] 'agreement among raft nodes before linearized reading' (duration: 2.202568997s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.061183Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:38.858176Z","time spent":"2.203000067s","remote":"127.0.0.1:41822","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/clusterroles\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:41.061340Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.522627422s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:41.061358Z","caller":"traceutil/trace.go:172","msg":"trace[1363201307] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:902; }","duration":"1.522647062s","start":"2026-03-30T21:30:39.538706Z","end":"2026-03-30T21:30:41.061353Z","steps":["trace[1363201307] 'agreement among raft nodes before linearized reading' (duration: 1.522619902s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.061350Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.520513109s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:41.061382Z","caller":"traceutil/trace.go:172","msg":"trace[904637899] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:902; }","duration":"1.52054336s","start":"2026-03-30T21:30:39.540830Z","end":"2026-03-30T21:30:41.061373Z","steps":["trace[904637899] 'agreement among raft nodes before linearized reading' (duration: 1.520493948s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.061405Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:39.540813Z","time spent":"1.52058742s","remote":"127.0.0.1:41158","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:30:41.516788Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"116.503805ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1.gateway.networking.k8s.io\" limit:1 ","response":"range_response_count:1 size:995"} {"level":"info","ts":"2026-03-30T21:30:41.516881Z","caller":"traceutil/trace.go:172","msg":"trace[1818353630] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1.gateway.networking.k8s.io; range_end:; response_count:1; response_revision:930; }","duration":"116.609128ms","start":"2026-03-30T21:30:41.400251Z","end":"2026-03-30T21:30:41.516860Z","steps":["trace[1818353630] 'agreement among raft nodes before linearized reading' (duration: 116.440813ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:41.517273Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"120.389812ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1beta1.gateway.networking.k8s.io\" limit:1 ","response":"range_response_count:1 size:1010"} {"level":"info","ts":"2026-03-30T21:30:41.517792Z","caller":"traceutil/trace.go:172","msg":"trace[793419632] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1beta1.gateway.networking.k8s.io; range_end:; response_count:1; response_revision:930; }","duration":"120.601237ms","start":"2026-03-30T21:30:41.396723Z","end":"2026-03-30T21:30:41.517324Z","steps":["trace[793419632] 'agreement among raft nodes before linearized reading' (duration: 120.155135ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:42.251448Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"354.426784ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:30:42.251547Z","caller":"traceutil/trace.go:172","msg":"trace[1739043942] transaction","detail":"{read_only:false; response_revision:939; number_of_response:1; }","duration":"626.98776ms","start":"2026-03-30T21:30:41.624541Z","end":"2026-03-30T21:30:42.251529Z","steps":["trace[1739043942] 'process raft request' (duration: 272.401802ms)","trace[1739043942] 'compare' (duration: 354.314401ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:42.251607Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:41.624523Z","time spent":"627.055312ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":23057,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:30:42.267335Z","caller":"traceutil/trace.go:172","msg":"trace[1795594034] transaction","detail":"{read_only:false; number_of_response:1; response_revision:940; }","duration":"627.438013ms","start":"2026-03-30T21:30:41.639881Z","end":"2026-03-30T21:30:42.267319Z","steps":["trace[1795594034] 'process raft request' (duration: 627.34242ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:42.267471Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:41.639865Z","time spent":"627.526575ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":105,"response count":0,"response size":29606,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:43.066191Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"428.118597ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:150508"} {"level":"warn","ts":"2026-03-30T21:30:43.066469Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"600.705915ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:43.066522Z","caller":"traceutil/trace.go:172","msg":"trace[1986906844] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:943; }","duration":"600.833838ms","start":"2026-03-30T21:30:42.465671Z","end":"2026-03-30T21:30:43.066505Z","steps":["trace[1986906844] 'range keys from in-memory index tree' (duration: 600.666643ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:43.066539Z","caller":"traceutil/trace.go:172","msg":"trace[67411123] transaction","detail":"{read_only:false; response_revision:944; number_of_response:1; }","duration":"801.841294ms","start":"2026-03-30T21:30:42.264684Z","end":"2026-03-30T21:30:43.066525Z","steps":["trace[67411123] 'process raft request' (duration: 801.597018ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:43.066610Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:42.264664Z","time spent":"801.909507ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":29452,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:30:43.066561Z","caller":"traceutil/trace.go:172","msg":"trace[1029488111] transaction","detail":"{read_only:false; number_of_response:1; response_revision:943; }","duration":"1.357739072s","start":"2026-03-30T21:30:41.708786Z","end":"2026-03-30T21:30:43.066525Z","steps":["trace[1029488111] 'process raft request' (duration: 929.209376ms)","trace[1029488111] 'compare' (duration: 427.713755ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:43.071771Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:41.708760Z","time spent":"1.358149262s","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":97,"response count":0,"response size":150531,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:43.350241Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360281,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:43.851099Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360281,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:44.081833Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.443720055s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T21:30:44.351309Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360281,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:44.494090Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"411.68715ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:30:44.494558Z","caller":"traceutil/trace.go:172","msg":"trace[904153377] transaction","detail":"{read_only:false; number_of_response:1; response_revision:948; }","duration":"2.205510063s","start":"2026-03-30T21:30:42.289034Z","end":"2026-03-30T21:30:44.494544Z","steps":["trace[904153377] 'process raft request' (duration: 2.205358849s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:44.494613Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:42.289018Z","time spent":"2.205568095s","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":112,"response count":0,"response size":23117,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:30:44.494652Z","caller":"traceutil/trace.go:172","msg":"trace[1188576676] transaction","detail":"{read_only:false; number_of_response:0; response_revision:948; }","duration":"2.210647232s","start":"2026-03-30T21:30:42.283969Z","end":"2026-03-30T21:30:44.494616Z","steps":["trace[1188576676] 'process raft request' (duration: 2.210411126s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:44.495772Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:42.283937Z","time spent":"2.210833776s","remote":"127.0.0.1:42244","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":28,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:30:44.584939Z","caller":"traceutil/trace.go:172","msg":"trace[36672578] linearizableReadLoop","detail":"{readStateIndex:1098; appliedIndex:1102; }","duration":"1.735415825s","start":"2026-03-30T21:30:42.849480Z","end":"2026-03-30T21:30:44.584896Z","steps":["trace[36672578] 'read index received' (duration: 1.735406835s)","trace[36672578] 'applied index is now lower than readState.Index' (duration: 7.41µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:44.585131Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.73562863s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:44.585168Z","caller":"traceutil/trace.go:172","msg":"trace[931673557] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:961; }","duration":"1.735681391s","start":"2026-03-30T21:30:42.849475Z","end":"2026-03-30T21:30:44.585156Z","steps":["trace[931673557] 'agreement among raft nodes before linearized reading' (duration: 1.735581469s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:44.585202Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:42.849454Z","time spent":"1.735740622s","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:30:44.585316Z","caller":"traceutil/trace.go:172","msg":"trace[1709458226] transaction","detail":"{read_only:false; response_revision:962; number_of_response:1; }","duration":"875.156411ms","start":"2026-03-30T21:30:43.710140Z","end":"2026-03-30T21:30:44.585296Z","steps":["trace[1709458226] 'process raft request' (duration: 875.030978ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:44.585455Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:43.710114Z","time spent":"875.274114ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:30:44.585642Z","caller":"traceutil/trace.go:172","msg":"trace[1977401291] transaction","detail":"{read_only:false; number_of_response:1; response_revision:961; }","duration":"1.50009628s","start":"2026-03-30T21:30:43.085525Z","end":"2026-03-30T21:30:44.585622Z","steps":["trace[1977401291] 'process raft request' (duration: 1.499553896s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:44.585884Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:43.085502Z","time spent":"1.500215882s","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":96,"response count":0,"response size":31181,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:44.607908Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360295,"received-request-id":16727104867003360281} {"level":"info","ts":"2026-03-30T21:30:44.639899Z","caller":"traceutil/trace.go:172","msg":"trace[279557706] transaction","detail":"{read_only:false; response_revision:963; number_of_response:1; }","duration":"634.527111ms","start":"2026-03-30T21:30:44.005346Z","end":"2026-03-30T21:30:44.639873Z","steps":["trace[279557706] 'process raft request' (duration: 602.669195ms)","trace[279557706] 'compare' (duration: 31.517467ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:44.640794Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:44.005318Z","time spent":"634.719065ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":675,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:44.888342Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"137.953128ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:30:44.888772Z","caller":"traceutil/trace.go:172","msg":"trace[650559050] transaction","detail":"{read_only:false; number_of_response:1; response_revision:965; }","duration":"378.898941ms","start":"2026-03-30T21:30:44.509855Z","end":"2026-03-30T21:30:44.888754Z","steps":["trace[650559050] 'process raft request' (duration: 378.770538ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:44.888870Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:44.509827Z","time spent":"378.988274ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":102,"response count":0,"response size":29512,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:45.086010Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360295,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:30:45.297933Z","caller":"traceutil/trace.go:172","msg":"trace[1289092054] linearizableReadLoop","detail":"{readStateIndex:1111; appliedIndex:1113; }","duration":"712.810733ms","start":"2026-03-30T21:30:44.585020Z","end":"2026-03-30T21:30:45.297831Z","steps":["trace[1289092054] 'read index received' (duration: 712.798253ms)","trace[1289092054] 'applied index is now lower than readState.Index' (duration: 10.87µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:45.298091Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.141319464s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"warn","ts":"2026-03-30T21:30:45.298107Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.767002256s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"warn","ts":"2026-03-30T21:30:45.298150Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.157874748s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:45.298203Z","caller":"traceutil/trace.go:172","msg":"trace[1753614898] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:966; }","duration":"1.767109319s","start":"2026-03-30T21:30:43.531071Z","end":"2026-03-30T21:30:45.298181Z","steps":["trace[1753614898] 'agreement among raft nodes before linearized reading' (duration: 1.766820132s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:45.298200Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"801.376057ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v1alpha1.gateway.envoyproxy.io\" limit:1 ","response":"range_response_count:1 size:1001"} {"level":"warn","ts":"2026-03-30T21:30:45.298253Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.231684571s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:30:45.298260Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:43.531058Z","time spent":"1.76718653s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"info","ts":"2026-03-30T21:30:45.298282Z","caller":"traceutil/trace.go:172","msg":"trace[1562439830] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v1alpha1.gateway.envoyproxy.io; range_end:; response_count:1; response_revision:966; }","duration":"801.47212ms","start":"2026-03-30T21:30:44.496790Z","end":"2026-03-30T21:30:45.298262Z","steps":["trace[1562439830] 'agreement among raft nodes before linearized reading' (duration: 801.215033ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:45.298294Z","caller":"traceutil/trace.go:172","msg":"trace[607873876] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:966; }","duration":"2.231718932s","start":"2026-03-30T21:30:43.066557Z","end":"2026-03-30T21:30:45.298276Z","steps":["trace[607873876] 'agreement among raft nodes before linearized reading' (duration: 2.231663781s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:45.298330Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:44.496770Z","time spent":"801.549841ms","remote":"127.0.0.1:42244","response type":"/etcdserverpb.KV/Range","request count":0,"request size":79,"response count":1,"response size":1024,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices/v1alpha1.gateway.envoyproxy.io\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:45.298478Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.970795525s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.78\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-30T21:30:45.298530Z","caller":"traceutil/trace.go:172","msg":"trace[1183157553] range","detail":"{range_begin:/registry/masterleases/199.204.45.78; range_end:; response_count:1; response_revision:966; }","duration":"1.970850147s","start":"2026-03-30T21:30:43.327666Z","end":"2026-03-30T21:30:45.298517Z","steps":["trace[1183157553] 'agreement among raft nodes before linearized reading' (duration: 1.97058577s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:45.298228Z","caller":"traceutil/trace.go:172","msg":"trace[649016829] range","detail":"{range_begin:/registry/resourceclaims; range_end:; response_count:0; response_revision:966; }","duration":"2.157952789s","start":"2026-03-30T21:30:43.140260Z","end":"2026-03-30T21:30:45.298212Z","steps":["trace[649016829] 'agreement among raft nodes before linearized reading' (duration: 2.157829526s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:45.298571Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:43.327652Z","time spent":"1.970907478s","remote":"127.0.0.1:41216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":1,"response size":156,"request content":"key:\"/registry/masterleases/199.204.45.78\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:45.298605Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:43.140247Z","time spent":"2.15834425s","remote":"127.0.0.1:42210","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/resourceclaims\" limit:1 "} {"level":"info","ts":"2026-03-30T21:30:45.298856Z","caller":"traceutil/trace.go:172","msg":"trace[1416949582] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:966; }","duration":"1.142092694s","start":"2026-03-30T21:30:44.156746Z","end":"2026-03-30T21:30:45.298839Z","steps":["trace[1416949582] 'agreement among raft nodes before linearized reading' (duration: 1.141193801s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:45.298894Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:44.156731Z","time spent":"1.142153106s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:45.298821Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.192614007s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:30:45.299905Z","caller":"traceutil/trace.go:172","msg":"trace[2111632344] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:966; }","duration":"2.193714704s","start":"2026-03-30T21:30:43.106177Z","end":"2026-03-30T21:30:45.299892Z","steps":["trace[2111632344] 'agreement among raft nodes before linearized reading' (duration: 2.19196353s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:45.299593Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.611628665s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-30T21:30:45.300198Z","caller":"traceutil/trace.go:172","msg":"trace[711432610] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:966; }","duration":"1.612259521s","start":"2026-03-30T21:30:43.687918Z","end":"2026-03-30T21:30:45.300178Z","steps":["trace[711432610] 'agreement among raft nodes before linearized reading' (duration: 1.610598429s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:45.300261Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:43.687892Z","time spent":"1.612353564s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":463,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:45.300325Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:43.106160Z","time spent":"2.193805036s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:45.332267Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360298,"received-request-id":16727104867003360295} {"level":"info","ts":"2026-03-30T21:30:45.370312Z","caller":"traceutil/trace.go:172","msg":"trace[1292571469] transaction","detail":"{read_only:false; number_of_response:1; response_revision:968; }","duration":"751.541953ms","start":"2026-03-30T21:30:44.618751Z","end":"2026-03-30T21:30:45.370293Z","steps":["trace[1292571469] 'process raft request' (duration: 751.368238ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:45.370443Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:44.618728Z","time spent":"751.641595ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":95,"response count":0,"response size":59667,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:30:45.370474Z","caller":"traceutil/trace.go:172","msg":"trace[458103622] transaction","detail":"{read_only:false; number_of_response:1; response_revision:970; }","duration":"449.785659ms","start":"2026-03-30T21:30:44.920676Z","end":"2026-03-30T21:30:45.370461Z","steps":["trace[458103622] 'process raft request' (duration: 449.734228ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:45.370649Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:44.920646Z","time spent":"449.949713ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":95,"response count":0,"response size":59667,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:45.468840Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"881.595086ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:45.468922Z","caller":"traceutil/trace.go:172","msg":"trace[719202729] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:971; }","duration":"881.738869ms","start":"2026-03-30T21:30:44.587164Z","end":"2026-03-30T21:30:45.468902Z","steps":["trace[719202729] 'agreement among raft nodes before linearized reading' (duration: 805.891412ms)","trace[719202729] 'range keys from in-memory index tree' (duration: 75.718185ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:45.468958Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:44.587142Z","time spent":"881.808482ms","remote":"127.0.0.1:41158","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:30:45.471442Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"171.227603ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:45.471605Z","caller":"traceutil/trace.go:172","msg":"trace[249826706] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:973; }","duration":"171.264934ms","start":"2026-03-30T21:30:45.300205Z","end":"2026-03-30T21:30:45.471470Z","steps":["trace[249826706] 'agreement among raft nodes before linearized reading' (duration: 171.216303ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:46.018766Z","caller":"traceutil/trace.go:172","msg":"trace[353952903] linearizableReadLoop","detail":"{readStateIndex:1146; appliedIndex:1154; }","duration":"169.936943ms","start":"2026-03-30T21:30:45.848814Z","end":"2026-03-30T21:30:46.018751Z","steps":["trace[353952903] 'read index received' (duration: 169.931063ms)","trace[353952903] 'applied index is now lower than readState.Index' (duration: 4.44µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:46.019064Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"170.23219ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:46.019168Z","caller":"traceutil/trace.go:172","msg":"trace[430009429] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:990; }","duration":"170.345893ms","start":"2026-03-30T21:30:45.848809Z","end":"2026-03-30T21:30:46.019155Z","steps":["trace[430009429] 'agreement among raft nodes before linearized reading' (duration: 170.165218ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:46.019858Z","caller":"traceutil/trace.go:172","msg":"trace[1518781218] transaction","detail":"{read_only:false; number_of_response:1; response_revision:989; }","duration":"323.196605ms","start":"2026-03-30T21:30:45.696588Z","end":"2026-03-30T21:30:46.019785Z","steps":["trace[1518781218] 'process raft request' (duration: 318.846696ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:46.020008Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:45.696563Z","time spent":"323.368991ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":105,"response count":0,"response size":74805,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:30:46.015646Z","caller":"traceutil/trace.go:172","msg":"trace[1747448989] transaction","detail":"{read_only:false; number_of_response:1; response_revision:989; }","duration":"308.402326ms","start":"2026-03-30T21:30:45.707227Z","end":"2026-03-30T21:30:46.015629Z","steps":["trace[1747448989] 'process raft request' (duration: 308.255852ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:46.022778Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:45.707204Z","time spent":"315.494634ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":105,"response count":0,"response size":74805,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:46.566363Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"101.532962ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:46.566519Z","caller":"traceutil/trace.go:172","msg":"trace[1135706235] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:996; }","duration":"101.654935ms","start":"2026-03-30T21:30:46.464795Z","end":"2026-03-30T21:30:46.566450Z","steps":["trace[1135706235] 'range keys from in-memory index tree' (duration: 101.48321ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:46.574099Z","caller":"traceutil/trace.go:172","msg":"trace[888627021] transaction","detail":"{read_only:false; number_of_response:1; response_revision:998; }","duration":"122.537408ms","start":"2026-03-30T21:30:46.451467Z","end":"2026-03-30T21:30:46.574005Z","steps":["trace[888627021] 'process raft request' (duration: 122.181629ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:47.225803Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360322,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:30:47.305313Z","caller":"traceutil/trace.go:172","msg":"trace[958633528] linearizableReadLoop","detail":"{readStateIndex:1172; appliedIndex:1173; }","duration":"580.215331ms","start":"2026-03-30T21:30:46.725077Z","end":"2026-03-30T21:30:47.305293Z","steps":["trace[958633528] 'read index received' (duration: 580.21048ms)","trace[958633528] 'applied index is now lower than readState.Index' (duration: 4.051µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:47.305473Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"580.358624ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/priorityclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:47.305503Z","caller":"traceutil/trace.go:172","msg":"trace[490484605] range","detail":"{range_begin:/registry/priorityclasses; range_end:; response_count:0; response_revision:1002; }","duration":"580.424456ms","start":"2026-03-30T21:30:46.725070Z","end":"2026-03-30T21:30:47.305495Z","steps":["trace[490484605] 'agreement among raft nodes before linearized reading' (duration: 580.327773ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:47.305532Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:46.725044Z","time spent":"580.481376ms","remote":"127.0.0.1:41842","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":28,"request content":"key:\"/registry/priorityclasses\" limit:1 "} {"level":"info","ts":"2026-03-30T21:30:47.305623Z","caller":"traceutil/trace.go:172","msg":"trace[1845942329] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1002; }","duration":"526.378743ms","start":"2026-03-30T21:30:46.779224Z","end":"2026-03-30T21:30:47.305603Z","steps":["trace[1845942329] 'process raft request' (duration: 526.172138ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:47.305822Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:46.779202Z","time spent":"526.530566ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":99,"response count":0,"response size":208521,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:47.307477Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"460.155165ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:47.307516Z","caller":"traceutil/trace.go:172","msg":"trace[902266947] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1002; }","duration":"460.194646ms","start":"2026-03-30T21:30:46.847312Z","end":"2026-03-30T21:30:47.307507Z","steps":["trace[902266947] 'agreement among raft nodes before linearized reading' (duration: 460.129625ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:47.307536Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:46.847292Z","time spent":"460.239688ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:30:47.307489Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"260.61457ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingwebhookconfigurations\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:47.307727Z","caller":"traceutil/trace.go:172","msg":"trace[56092378] range","detail":"{range_begin:/registry/validatingwebhookconfigurations; range_end:; response_count:0; response_revision:1002; }","duration":"260.855006ms","start":"2026-03-30T21:30:47.046864Z","end":"2026-03-30T21:30:47.307719Z","steps":["trace[56092378] 'agreement among raft nodes before linearized reading' (duration: 260.597459ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:47.728869Z","caller":"traceutil/trace.go:172","msg":"trace[1803473418] linearizableReadLoop","detail":"{readStateIndex:1178; appliedIndex:1178; }","duration":"181.18879ms","start":"2026-03-30T21:30:47.547647Z","end":"2026-03-30T21:30:47.728835Z","steps":["trace[1803473418] 'read index received' (duration: 181.181979ms)","trace[1803473418] 'applied index is now lower than readState.Index' (duration: 5.281µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:47.729081Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"181.410835ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:47.729123Z","caller":"traceutil/trace.go:172","msg":"trace[821836065] range","detail":"{range_begin:/registry/pods; range_end:; response_count:0; response_revision:1005; }","duration":"181.469746ms","start":"2026-03-30T21:30:47.547643Z","end":"2026-03-30T21:30:47.729112Z","steps":["trace[821836065] 'agreement among raft nodes before linearized reading' (duration: 181.368954ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:48.582837Z","caller":"traceutil/trace.go:172","msg":"trace[179146766] linearizableReadLoop","detail":"{readStateIndex:1178; appliedIndex:1179; }","duration":"117.457015ms","start":"2026-03-30T21:30:48.465356Z","end":"2026-03-30T21:30:48.582813Z","steps":["trace[179146766] 'read index received' (duration: 117.446645ms)","trace[179146766] 'applied index is now lower than readState.Index' (duration: 8.77µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:48.582953Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"117.578268ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:48.583005Z","caller":"traceutil/trace.go:172","msg":"trace[1965834393] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1007; }","duration":"117.64393ms","start":"2026-03-30T21:30:48.465350Z","end":"2026-03-30T21:30:48.582994Z","steps":["trace[1965834393] 'agreement among raft nodes before linearized reading' (duration: 117.549987ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:49.881345Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360339,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:50.282653Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"614.033501ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-30T21:30:50.382077Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360339,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:50.573548Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"290.4085ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:30:50.654713Z","caller":"traceutil/trace.go:172","msg":"trace[2058398830] linearizableReadLoop","detail":"{readStateIndex:1188; appliedIndex:1186; }","duration":"1.274154848s","start":"2026-03-30T21:30:49.380534Z","end":"2026-03-30T21:30:50.654689Z","steps":["trace[2058398830] 'read index received' (duration: 1.221983779s)","trace[2058398830] 'applied index is now lower than readState.Index' (duration: 52.170269ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:50.654810Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360340,"received-request-id":16727104867003360339} {"level":"warn","ts":"2026-03-30T21:30:50.654861Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360340,"received-request-id":16727104867003360339} {"level":"warn","ts":"2026-03-30T21:30:50.655142Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.27460375s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T21:30:50.655184Z","caller":"traceutil/trace.go:172","msg":"trace[1777820827] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1015; }","duration":"1.274663331s","start":"2026-03-30T21:30:49.380509Z","end":"2026-03-30T21:30:50.655173Z","steps":["trace[1777820827] 'agreement among raft nodes before linearized reading' (duration: 1.274477706s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:50.655218Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:49.380487Z","time spent":"1.274722083s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":506,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:50.656752Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"806.872726ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:30:50.656749Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.191006302s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:50.656800Z","caller":"traceutil/trace.go:172","msg":"trace[617207219] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1015; }","duration":"806.921977ms","start":"2026-03-30T21:30:49.849866Z","end":"2026-03-30T21:30:50.656788Z","steps":["trace[617207219] 'agreement among raft nodes before linearized reading' (duration: 806.844665ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:50.656783Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"386.425842ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:50.656830Z","caller":"traceutil/trace.go:172","msg":"trace[963602040] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1015; }","duration":"1.191101415s","start":"2026-03-30T21:30:49.465710Z","end":"2026-03-30T21:30:50.656811Z","steps":["trace[963602040] 'agreement among raft nodes before linearized reading' (duration: 1.190972962s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:50.656897Z","caller":"traceutil/trace.go:172","msg":"trace[1899127786] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:0; response_revision:1015; }","duration":"386.551455ms","start":"2026-03-30T21:30:50.270321Z","end":"2026-03-30T21:30:50.656872Z","steps":["trace[1899127786] 'agreement among raft nodes before linearized reading' (duration: 386.35981ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:50.656977Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:50.270296Z","time spent":"386.659988ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":62,"response count":0,"response size":28,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:50.656831Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:49.849845Z","time spent":"806.978829ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:30:50.890123Z","caller":"traceutil/trace.go:172","msg":"trace[653486130] transaction","detail":"{read_only:false; number_of_response:0; response_revision:1023; }","duration":"100.558164ms","start":"2026-03-30T21:30:50.789545Z","end":"2026-03-30T21:30:50.890103Z","steps":["trace[653486130] 'process raft request' (duration: 100.087792ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:50.890371Z","caller":"traceutil/trace.go:172","msg":"trace[1320024079] transaction","detail":"{read_only:false; number_of_response:0; response_revision:1023; }","duration":"100.535764ms","start":"2026-03-30T21:30:50.789825Z","end":"2026-03-30T21:30:50.890361Z","steps":["trace[1320024079] 'process raft request' (duration: 99.893318ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:50.995768Z","caller":"traceutil/trace.go:172","msg":"trace[1059517256] linearizableReadLoop","detail":"{readStateIndex:1212; appliedIndex:1214; }","duration":"104.079543ms","start":"2026-03-30T21:30:50.891664Z","end":"2026-03-30T21:30:50.995743Z","steps":["trace[1059517256] 'read index received' (duration: 104.071412ms)","trace[1059517256] 'applied index is now lower than readState.Index' (duration: 6.801µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:50.996066Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"104.3282ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v2alpha1.cilium.io\" limit:1 ","response":"range_response_count:1 size:965"} {"level":"info","ts":"2026-03-30T21:30:50.996169Z","caller":"traceutil/trace.go:172","msg":"trace[1812528948] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v2alpha1.cilium.io; range_end:; response_count:1; response_revision:1028; }","duration":"104.490904ms","start":"2026-03-30T21:30:50.891659Z","end":"2026-03-30T21:30:50.996150Z","steps":["trace[1812528948] 'agreement among raft nodes before linearized reading' (duration: 104.188436ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:51.033459Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"141.642557ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/v2.cilium.io\" limit:1 ","response":"range_response_count:1 size:947"} {"level":"info","ts":"2026-03-30T21:30:51.033619Z","caller":"traceutil/trace.go:172","msg":"trace[345801330] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/v2.cilium.io; range_end:; response_count:1; response_revision:1029; }","duration":"141.856012ms","start":"2026-03-30T21:30:50.891746Z","end":"2026-03-30T21:30:51.033602Z","steps":["trace[345801330] 'agreement among raft nodes before linearized reading' (duration: 141.446421ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:51.756956Z","caller":"traceutil/trace.go:172","msg":"trace[283336068] linearizableReadLoop","detail":"{readStateIndex:1228; appliedIndex:1229; }","duration":"459.539975ms","start":"2026-03-30T21:30:51.297386Z","end":"2026-03-30T21:30:51.756926Z","steps":["trace[283336068] 'read index received' (duration: 459.525105ms)","trace[283336068] 'applied index is now lower than readState.Index' (duration: 13.18µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:51.757284Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"459.873394ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:30:51.757390Z","caller":"traceutil/trace.go:172","msg":"trace[467310092] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1037; }","duration":"459.994117ms","start":"2026-03-30T21:30:51.297374Z","end":"2026-03-30T21:30:51.757368Z","steps":["trace[467310092] 'agreement among raft nodes before linearized reading' (duration: 459.69779ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:51.757443Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:51.297337Z","time spent":"460.0945ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T21:30:51.758342Z","caller":"traceutil/trace.go:172","msg":"trace[66375523] transaction","detail":"{read_only:false; response_revision:1039; number_of_response:1; }","duration":"470.0852ms","start":"2026-03-30T21:30:51.288236Z","end":"2026-03-30T21:30:51.758321Z","steps":["trace[66375523] 'process raft request' (duration: 469.967327ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:51.758463Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:51.288197Z","time spent":"470.195113ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18506,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:52.257960Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360364,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:52.758436Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360364,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:53.259577Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360364,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:30:53.273950Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"45496929f16fe822 is starting a new election at term 2"} {"level":"info","ts":"2026-03-30T21:30:53.274019Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"45496929f16fe822 became pre-candidate at term 2"} {"level":"info","ts":"2026-03-30T21:30:53.274051Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"45496929f16fe822 [logterm: 2, index: 1234] sent MsgPreVote request to 73b132709fee1095 at term 2"} {"level":"info","ts":"2026-03-30T21:30:53.274068Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"45496929f16fe822 [logterm: 2, index: 1234] sent MsgPreVote request to d98d5f2c752cc52d at term 2"} {"level":"info","ts":"2026-03-30T21:30:53.274086Z","logger":"raft","caller":"v3@v3.6.0/node.go:376","msg":"raft.node: 45496929f16fe822 lost leader d98d5f2c752cc52d at term 2"} {"level":"info","ts":"2026-03-30T21:30:53.274167Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"45496929f16fe822 received MsgPreVoteResp from 45496929f16fe822 at term 2"} {"level":"info","ts":"2026-03-30T21:30:53.274187Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"45496929f16fe822 has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-30T21:30:53.760496Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360364,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:53.848434Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999851881s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-30T21:30:53.848607Z","caller":"traceutil/trace.go:172","msg":"trace[1332974968] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000046526s","start":"2026-03-30T21:30:51.848535Z","end":"2026-03-30T21:30:53.848581Z","steps":["trace[1332974968] 'agreement among raft nodes before linearized reading' (duration: 1.999846471s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:53.848680Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:51.848512Z","time spent":"2.000154709s","remote":"127.0.0.1:41158","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 21:30:53 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-30T21:30:53.935579Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"45496929f16fe822 became follower at term 2"} {"level":"info","ts":"2026-03-30T21:30:53.935632Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: 45496929f16fe822 elected leader d98d5f2c752cc52d at term 2"} {"level":"warn","ts":"2026-03-30T21:30:53.935828Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.469863794s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-30T21:30:53.935831Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.338685077s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-30T21:30:53.935871Z","caller":"traceutil/trace.go:172","msg":"trace[1217101556] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; }","duration":"2.338731128s","start":"2026-03-30T21:30:51.597131Z","end":"2026-03-30T21:30:53.935862Z","steps":["trace[1217101556] 'agreement among raft nodes before linearized reading' (duration: 2.338683517s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:53.935871Z","caller":"traceutil/trace.go:172","msg":"trace[1295608260] range","detail":"{range_begin:; range_end:; }","duration":"2.469935085s","start":"2026-03-30T21:30:51.465923Z","end":"2026-03-30T21:30:53.935858Z","steps":["trace[1295608260] 'agreement among raft nodes before linearized reading' (duration: 2.469861354s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:53.935908Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:51.597113Z","time spent":"2.33878681s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"error","ts":"2026-03-30T21:30:53.935921Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[+]non_learner ok\n[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: leader changed\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-30T21:30:54.251428Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360366,"received-request-id":16727104867003360364} {"level":"info","ts":"2026-03-30T21:30:54.296305Z","caller":"traceutil/trace.go:172","msg":"trace[951706289] linearizableReadLoop","detail":"{readStateIndex:1234; appliedIndex:1242; }","duration":"360.510895ms","start":"2026-03-30T21:30:53.935771Z","end":"2026-03-30T21:30:54.296281Z","steps":["trace[951706289] 'read index received' (duration: 360.419102ms)","trace[951706289] 'applied index is now lower than readState.Index' (duration: 90.363µs)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:30:54.296501Z","caller":"traceutil/trace.go:172","msg":"trace[804494421] transaction","detail":"{read_only:false; response_revision:1051; number_of_response:1; }","duration":"434.612859ms","start":"2026-03-30T21:30:53.861860Z","end":"2026-03-30T21:30:54.296472Z","steps":["trace[804494421] 'process raft request' (duration: 434.398054ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.296569Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.505106238s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 ","response":"range_response_count:1 size:495"} {"level":"warn","ts":"2026-03-30T21:30:54.296570Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"968.416118ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.78\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-30T21:30:54.296604Z","caller":"traceutil/trace.go:172","msg":"trace[227143660] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:1051; }","duration":"1.505161609s","start":"2026-03-30T21:30:52.791433Z","end":"2026-03-30T21:30:54.296595Z","steps":["trace[227143660] 'agreement among raft nodes before linearized reading' (duration: 1.504991335s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:54.296623Z","caller":"traceutil/trace.go:172","msg":"trace[1609605188] range","detail":"{range_begin:/registry/masterleases/199.204.45.78; range_end:; response_count:1; response_revision:1051; }","duration":"968.47904ms","start":"2026-03-30T21:30:53.328132Z","end":"2026-03-30T21:30:54.296611Z","steps":["trace[1609605188] 'agreement among raft nodes before linearized reading' (duration: 968.262564ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.296626Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:53.861841Z","time spent":"434.727642ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:30:54.296645Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:52.791410Z","time spent":"1.505220401s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":62,"response count":1,"response size":518,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:54.296656Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:53.328110Z","time spent":"968.536632ms","remote":"127.0.0.1:41216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":1,"response size":156,"request content":"key:\"/registry/masterleases/199.204.45.78\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:54.296958Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.419386063s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T21:30:54.296994Z","caller":"traceutil/trace.go:172","msg":"trace[1120892531] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1051; }","duration":"2.419422404s","start":"2026-03-30T21:30:51.877562Z","end":"2026-03-30T21:30:54.296985Z","steps":["trace[1120892531] 'agreement among raft nodes before linearized reading' (duration: 2.419031903s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.297019Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:51.877549Z","time spent":"2.419463075s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:54.297093Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"434.451795ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:54.297131Z","caller":"traceutil/trace.go:172","msg":"trace[971787582] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1051; }","duration":"434.493226ms","start":"2026-03-30T21:30:53.862627Z","end":"2026-03-30T21:30:54.297120Z","steps":["trace[971787582] 'agreement among raft nodes before linearized reading' (duration: 433.919031ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.297158Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:53.862609Z","time spent":"434.542657ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:30:54.297238Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"716.811729ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterrolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:30:54.297193Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.363760035s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/roles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:54.297282Z","caller":"traceutil/trace.go:172","msg":"trace[836893075] range","detail":"{range_begin:/registry/clusterrolebindings; range_end:; response_count:0; response_revision:1051; }","duration":"716.85608ms","start":"2026-03-30T21:30:53.580412Z","end":"2026-03-30T21:30:54.297268Z","steps":["trace[836893075] 'agreement among raft nodes before linearized reading' (duration: 716.43356ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:30:54.297303Z","caller":"traceutil/trace.go:172","msg":"trace[1182824582] range","detail":"{range_begin:/registry/roles; range_end:; response_count:0; response_revision:1051; }","duration":"2.363872878s","start":"2026-03-30T21:30:51.933415Z","end":"2026-03-30T21:30:54.297288Z","steps":["trace[1182824582] 'agreement among raft nodes before linearized reading' (duration: 2.36319844s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.297310Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:53.580389Z","time spent":"716.914172ms","remote":"127.0.0.1:41830","response type":"/etcdserverpb.KV/Range","request count":0,"request size":33,"response count":0,"response size":28,"request content":"key:\"/registry/clusterrolebindings\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:54.297335Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:51.933380Z","time spent":"2.36394584s","remote":"127.0.0.1:41798","response type":"/etcdserverpb.KV/Range","request count":0,"request size":19,"response count":0,"response size":28,"request content":"key:\"/registry/roles\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:54.297690Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.924509233s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumes\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:54.297761Z","caller":"traceutil/trace.go:172","msg":"trace[41143052] range","detail":"{range_begin:/registry/persistentvolumes; range_end:; response_count:0; response_revision:1051; }","duration":"1.924580935s","start":"2026-03-30T21:30:52.373168Z","end":"2026-03-30T21:30:54.297749Z","steps":["trace[41143052] 'agreement among raft nodes before linearized reading' (duration: 1.923436936s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.297924Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:52.373144Z","time spent":"1.924648566s","remote":"127.0.0.1:41408","response type":"/etcdserverpb.KV/Range","request count":0,"request size":31,"response count":0,"response size":28,"request content":"key:\"/registry/persistentvolumes\" limit:1 "} {"level":"info","ts":"2026-03-30T21:30:54.298285Z","caller":"traceutil/trace.go:172","msg":"trace[1409010564] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1054; }","duration":"355.671494ms","start":"2026-03-30T21:30:53.942588Z","end":"2026-03-30T21:30:54.298259Z","steps":["trace[1409010564] 'process raft request' (duration: 355.561841ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.298421Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:53.942568Z","time spent":"355.777846ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":88,"response count":0,"response size":5202,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:30:54.299053Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"361.655734ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:54.299092Z","caller":"traceutil/trace.go:172","msg":"trace[950495530] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1054; }","duration":"361.696225ms","start":"2026-03-30T21:30:53.937387Z","end":"2026-03-30T21:30:54.299083Z","steps":["trace[950495530] 'agreement among raft nodes before linearized reading' (duration: 361.636134ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.299299Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"233.855577ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T21:30:54.299323Z","caller":"traceutil/trace.go:172","msg":"trace[1938805784] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1054; }","duration":"233.881538ms","start":"2026-03-30T21:30:54.065434Z","end":"2026-03-30T21:30:54.299316Z","steps":["trace[1938805784] 'agreement among raft nodes before linearized reading' (duration: 233.832957ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.300658Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"336.971053ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:30:54.300731Z","caller":"traceutil/trace.go:172","msg":"trace[1202128078] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1054; }","duration":"337.099746ms","start":"2026-03-30T21:30:53.963617Z","end":"2026-03-30T21:30:54.300717Z","steps":["trace[1202128078] 'agreement among raft nodes before linearized reading' (duration: 335.72342ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:54.300769Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:53.963594Z","time spent":"337.160878ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:30:54.800225Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360369,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:55.300874Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360369,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:55.801060Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360369,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:30:55.974221Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"45496929f16fe822 [logterm: 2, index: 1259, vote: 0] ignored MsgPreVote from 73b132709fee1095 [logterm: 2, index: 1259] at term 2: lease is not expired (remaining ticks: 9)"} {"level":"warn","ts":"2026-03-30T21:30:56.299553Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.00004982s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T21:30:56.299652Z","caller":"traceutil/trace.go:172","msg":"trace[1157485980] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000187433s","start":"2026-03-30T21:30:54.299442Z","end":"2026-03-30T21:30:56.299629Z","steps":["trace[1157485980] 'agreement among raft nodes before linearized reading' (duration: 2.000045659s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:56.299754Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:54.299432Z","time spent":"2.000307176s","remote":"127.0.0.1:41158","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 21:30:56 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T21:30:56.301703Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360369,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:56.802264Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360369,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:57.303154Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360369,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:57.803730Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360369,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:57.976482Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"3.504591477s","expected-duration":"1s"} {"level":"info","ts":"2026-03-30T21:30:57.976878Z","caller":"traceutil/trace.go:172","msg":"trace[611072526] linearizableReadLoop","detail":"{readStateIndex:1248; appliedIndex:1248; }","duration":"3.677399458s","start":"2026-03-30T21:30:54.299446Z","end":"2026-03-30T21:30:57.976845Z","steps":["trace[611072526] 'read index received' (duration: 3.677387238s)","trace[611072526] 'applied index is now lower than readState.Index' (duration: 9.6µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:30:58.236992Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.770724612s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:30:58.237076Z","caller":"traceutil/trace.go:172","msg":"trace[704393944] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1054; }","duration":"3.771533932s","start":"2026-03-30T21:30:54.465524Z","end":"2026-03-30T21:30:58.237058Z","steps":["trace[704393944] 'range keys from in-memory index tree' (duration: 3.77067099s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:58.237115Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.765232314s","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:40"} {"level":"warn","ts":"2026-03-30T21:30:58.237315Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:54.299104Z","time spent":"3.938208209s","remote":"127.0.0.1:41216","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-30T21:30:58.237863Z","caller":"traceutil/trace.go:172","msg":"trace[1673542524] transaction","detail":"{read_only:false; response_revision:1055; number_of_response:1; }","duration":"3.937016409s","start":"2026-03-30T21:30:54.300827Z","end":"2026-03-30T21:30:58.237844Z","steps":["trace[1673542524] 'process raft request' (duration: 3.936387384s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:58.237952Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:54.300799Z","time spent":"3.937110721s","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":739,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:30:58.307793Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999840237s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-30T21:30:58.307895Z","caller":"traceutil/trace.go:172","msg":"trace[717728824] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.999977101s","start":"2026-03-30T21:30:56.307896Z","end":"2026-03-30T21:30:58.307874Z","steps":["trace[717728824] 'agreement among raft nodes before linearized reading' (duration: 1.999836508s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:30:58.307961Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:56.307867Z","time spent":"2.000081803s","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:30:58.477914Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360378,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:58.963013Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"242.024387ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T21:30:58.978729Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360378,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:59.308047Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:54.309203Z","time spent":"4.998837932s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} 2026/03/30 21:30:59 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T21:30:59.479203Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360378,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:30:59.979969Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360378,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:31:00.308744Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999870869s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T21:31:00.309044Z","caller":"traceutil/trace.go:172","msg":"trace[792464991] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000148946s","start":"2026-03-30T21:30:58.308844Z","end":"2026-03-30T21:31:00.308993Z","steps":["trace[792464991] 'agreement among raft nodes before linearized reading' (duration: 1.999868139s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.309189Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:58.308790Z","time spent":"2.000383061s","remote":"127.0.0.1:41158","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 21:31:00 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T21:31:00.480481Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360378,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:31:00.832772Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"2.111974474s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T21:31:00.833465Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.282152484s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:00.834018Z","caller":"traceutil/trace.go:172","msg":"trace[228755628] linearizableReadLoop","detail":"{readStateIndex:1263; appliedIndex:1263; }","duration":"2.856984697s","start":"2026-03-30T21:30:57.977006Z","end":"2026-03-30T21:31:00.833991Z","steps":["trace[228755628] 'read index received' (duration: 2.856976467s)","trace[228755628] 'applied index is now lower than readState.Index' (duration: 7.13µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:00.834190Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.222697781s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-30T21:31:00.834230Z","caller":"traceutil/trace.go:172","msg":"trace[89721563] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1065; }","duration":"3.222749022s","start":"2026-03-30T21:30:57.611470Z","end":"2026-03-30T21:31:00.834219Z","steps":["trace[89721563] 'agreement among raft nodes before linearized reading' (duration: 3.22264438s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.834263Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:57.611447Z","time spent":"3.222807634s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:00.834313Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.709325747s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:00.834383Z","caller":"traceutil/trace.go:172","msg":"trace[421929422] range","detail":"{range_begin:/registry/horizontalpodautoscalers; range_end:; response_count:0; response_revision:1065; }","duration":"3.709398879s","start":"2026-03-30T21:30:57.124968Z","end":"2026-03-30T21:31:00.834366Z","steps":["trace[421929422] 'agreement among raft nodes before linearized reading' (duration: 3.709281766s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.834461Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.536747233s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:00.834552Z","caller":"traceutil/trace.go:172","msg":"trace[523088568] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices; range_end:; response_count:0; response_revision:1065; }","duration":"3.536842375s","start":"2026-03-30T21:30:57.297693Z","end":"2026-03-30T21:31:00.834535Z","steps":["trace[523088568] 'agreement among raft nodes before linearized reading' (duration: 3.53664604s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.834601Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.884809468s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:31:00.834639Z","caller":"traceutil/trace.go:172","msg":"trace[527190214] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1065; }","duration":"4.884846059s","start":"2026-03-30T21:30:55.949784Z","end":"2026-03-30T21:31:00.834630Z","steps":["trace[527190214] 'agreement among raft nodes before linearized reading' (duration: 4.884732526s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.834666Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:55.949757Z","time spent":"4.884902159s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:00.834618Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:57.297659Z","time spent":"3.536946838s","remote":"127.0.0.1:42244","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":28,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:00.834494Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.171021957s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-30T21:31:00.834927Z","caller":"traceutil/trace.go:172","msg":"trace[455491675] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1065; }","duration":"4.171446338s","start":"2026-03-30T21:30:56.663461Z","end":"2026-03-30T21:31:00.834908Z","steps":["trace[455491675] 'agreement among raft nodes before linearized reading' (duration: 4.170999406s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.834987Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:56.663437Z","time spent":"4.171531819s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:00.834494Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:57.124946Z","time spent":"3.709536622s","remote":"127.0.0.1:41562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":38,"response count":0,"response size":28,"request content":"key:\"/registry/horizontalpodautoscalers\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:00.835817Z","caller":"traceutil/trace.go:172","msg":"trace[1751439164] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1065; }","duration":"1.284508464s","start":"2026-03-30T21:30:59.551284Z","end":"2026-03-30T21:31:00.835792Z","steps":["trace[1751439164] 'range keys from in-memory index tree' (duration: 1.282100263s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:00.837376Z","caller":"traceutil/trace.go:172","msg":"trace[83261080] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1069; }","duration":"6.532748213s","start":"2026-03-30T21:30:54.304612Z","end":"2026-03-30T21:31:00.837360Z","steps":["trace[83261080] 'process raft request' (duration: 6.531901262s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.837526Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:54.304589Z","time spent":"6.532831915s","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":88,"response count":0,"response size":5211,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:00.837792Z","caller":"traceutil/trace.go:172","msg":"trace[80471397] transaction","detail":"{read_only:false; response_revision:1072; number_of_response:1; }","duration":"6.050935963s","start":"2026-03-30T21:30:54.786816Z","end":"2026-03-30T21:31:00.837752Z","steps":["trace[80471397] 'process raft request' (duration: 6.050840541s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:00.838080Z","caller":"traceutil/trace.go:172","msg":"trace[1369465544] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1069; }","duration":"6.530462116s","start":"2026-03-30T21:30:54.307218Z","end":"2026-03-30T21:31:00.837680Z","steps":["trace[1369465544] 'process raft request' (duration: 6.52982388s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.838118Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:54.786791Z","time spent":"6.051128018s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:00.837878Z","caller":"traceutil/trace.go:172","msg":"trace[2132519226] transaction","detail":"{read_only:false; response_revision:1071; number_of_response:1; }","duration":"6.241429819s","start":"2026-03-30T21:30:54.596432Z","end":"2026-03-30T21:31:00.837862Z","steps":["trace[2132519226] 'process raft request' (duration: 6.241185953s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.838248Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:54.596409Z","time spent":"6.241797389s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:00.838269Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:54.307206Z","time spent":"6.530979119s","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":99,"response count":0,"response size":10411,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:00.838384Z","caller":"traceutil/trace.go:172","msg":"trace[1818420700] transaction","detail":"{read_only:false; response_revision:1073; number_of_response:1; }","duration":"2.600112505s","start":"2026-03-30T21:30:58.238259Z","end":"2026-03-30T21:31:00.838372Z","steps":["trace[1818420700] 'process raft request' (duration: 2.599459818s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:00.838492Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:58.238236Z","time spent":"2.600189437s","remote":"127.0.0.1:41216","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":118,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:01.335142Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360380,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:31:01.836201Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360380,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:31:02.009244Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.174136347s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T21:31:02.042356Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360380,"received-request-id":16727104867003360378} {"level":"info","ts":"2026-03-30T21:31:02.126133Z","caller":"traceutil/trace.go:172","msg":"trace[1370911768] linearizableReadLoop","detail":"{readStateIndex:1312; appliedIndex:1302; }","duration":"1.292023068s","start":"2026-03-30T21:31:00.834086Z","end":"2026-03-30T21:31:02.126109Z","steps":["trace[1370911768] 'read index received' (duration: 1.271392566s)","trace[1370911768] 'applied index is now lower than readState.Index' (duration: 20.628842ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:02.126365Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.572320221s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/poddisruptionbudgets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:02.126410Z","caller":"traceutil/trace.go:172","msg":"trace[882306237] range","detail":"{range_begin:/registry/poddisruptionbudgets; range_end:; response_count:0; response_revision:1099; }","duration":"1.572412003s","start":"2026-03-30T21:31:00.553988Z","end":"2026-03-30T21:31:02.126400Z","steps":["trace[882306237] 'agreement among raft nodes before linearized reading' (duration: 1.572233259s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:02.126589Z","caller":"traceutil/trace.go:172","msg":"trace[1050530359] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1101; }","duration":"1.274760051s","start":"2026-03-30T21:31:00.851814Z","end":"2026-03-30T21:31:02.126575Z","steps":["trace[1050530359] 'process raft request' (duration: 1.274674309s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.126492Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:00.553970Z","time spent":"1.572507475s","remote":"127.0.0.1:41782","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":28,"request content":"key:\"/registry/poddisruptionbudgets\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:02.126850Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:00.851796Z","time spent":"1.274871125s","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":82,"response count":0,"response size":24280,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:02.126907Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.884834058s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a1bbd58260c712\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-30T21:31:02.126996Z","caller":"traceutil/trace.go:172","msg":"trace[1449049170] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18a1bbd58260c712; range_end:; response_count:1; response_revision:1102; }","duration":"3.88493857s","start":"2026-03-30T21:30:58.242037Z","end":"2026-03-30T21:31:02.126975Z","steps":["trace[1449049170] 'agreement among raft nodes before linearized reading' (duration: 3.884648193s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.127038Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:58.242021Z","time spent":"3.885006642s","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":817,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a1bbd58260c712\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:02.127134Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.81529687s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 ","response":"range_response_count:1 size:7806"} {"level":"warn","ts":"2026-03-30T21:31:02.127119Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.812244704s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:02.127201Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.890078101s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:02.127227Z","caller":"traceutil/trace.go:172","msg":"trace[1271337854] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1102; }","duration":"3.890109401s","start":"2026-03-30T21:30:58.237111Z","end":"2026-03-30T21:31:02.127220Z","steps":["trace[1271337854] 'agreement among raft nodes before linearized reading' (duration: 3.89007007s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:02.127225Z","caller":"traceutil/trace.go:172","msg":"trace[1449058514] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-2; range_end:; response_count:1; response_revision:1102; }","duration":"1.815404093s","start":"2026-03-30T21:31:00.311799Z","end":"2026-03-30T21:31:02.127203Z","steps":["trace[1449058514] 'agreement among raft nodes before linearized reading' (duration: 1.815116796s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.127251Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.853066654s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattributesclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:02.127236Z","caller":"traceutil/trace.go:172","msg":"trace[1181538549] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1102; }","duration":"1.812417148s","start":"2026-03-30T21:31:00.314799Z","end":"2026-03-30T21:31:02.127216Z","steps":["trace[1181538549] 'agreement among raft nodes before linearized reading' (duration: 1.812154152s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.127269Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:00.311786Z","time spent":"1.815473485s","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":1,"response size":7829,"request content":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:02.127304Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:00.314794Z","time spent":"1.8125017s","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:31:02.127289Z","caller":"traceutil/trace.go:172","msg":"trace[1334243432] range","detail":"{range_begin:/registry/volumeattributesclasses; range_end:; response_count:0; response_revision:1102; }","duration":"2.853097516s","start":"2026-03-30T21:30:59.274177Z","end":"2026-03-30T21:31:02.127274Z","steps":["trace[1334243432] 'agreement among raft nodes before linearized reading' (duration: 2.853053305s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.127365Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:59.274145Z","time spent":"2.853210478s","remote":"127.0.0.1:41902","response type":"/etcdserverpb.KV/Range","request count":0,"request size":37,"response count":0,"response size":28,"request content":"key:\"/registry/volumeattributesclasses\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:02.127161Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.715480964s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:31:02.127428Z","caller":"traceutil/trace.go:172","msg":"trace[162977860] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1102; }","duration":"3.71575202s","start":"2026-03-30T21:30:58.411663Z","end":"2026-03-30T21:31:02.127415Z","steps":["trace[162977860] 'agreement among raft nodes before linearized reading' (duration: 3.715293689s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.127470Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:58.411639Z","time spent":"3.715818582s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:02.128380Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.340320955s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/networkpolicies\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:02.128448Z","caller":"traceutil/trace.go:172","msg":"trace[1061943941] range","detail":"{range_begin:/registry/networkpolicies; range_end:; response_count:0; response_revision:1102; }","duration":"3.340393737s","start":"2026-03-30T21:30:58.788041Z","end":"2026-03-30T21:31:02.128434Z","steps":["trace[1061943941] 'agreement among raft nodes before linearized reading' (duration: 3.338641433s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.128482Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:30:58.788019Z","time spent":"3.340453279s","remote":"127.0.0.1:41680","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":28,"request content":"key:\"/registry/networkpolicies\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:02.626594Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360382,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:02.755764Z","caller":"traceutil/trace.go:172","msg":"trace[243351270] linearizableReadLoop","detail":"{readStateIndex:1320; appliedIndex:1321; }","duration":"629.537488ms","start":"2026-03-30T21:31:02.126196Z","end":"2026-03-30T21:31:02.755733Z","steps":["trace[243351270] 'read index received' (duration: 629.525257ms)","trace[243351270] 'applied index is now lower than readState.Index' (duration: 10.671µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:02.756089Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.914986359s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-03-30T21:31:02.756112Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.910860615s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/ciliuml2announcementpolicies.cilium.io\" limit:1 ","response":"range_response_count:1 size:10377"} {"level":"info","ts":"2026-03-30T21:31:02.756161Z","caller":"traceutil/trace.go:172","msg":"trace[1566037956] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1106; }","duration":"1.915073881s","start":"2026-03-30T21:31:00.841072Z","end":"2026-03-30T21:31:02.756145Z","steps":["trace[1566037956] 'agreement among raft nodes before linearized reading' (duration: 1.914819105s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:02.756203Z","caller":"traceutil/trace.go:172","msg":"trace[2001286061] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions/ciliuml2announcementpolicies.cilium.io; range_end:; response_count:1; response_revision:1106; }","duration":"1.910969408s","start":"2026-03-30T21:31:00.845211Z","end":"2026-03-30T21:31:02.756180Z","steps":["trace[2001286061] 'agreement among raft nodes before linearized reading' (duration: 1.91066584s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.756089Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.446020794s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 ","response":"range_response_count:1 size:495"} {"level":"warn","ts":"2026-03-30T21:31:02.756214Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:00.841057Z","time spent":"1.915141282s","remote":"127.0.0.1:41448","response type":"/etcdserverpb.KV/Range","request count":0,"request size":51,"response count":1,"response size":464,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:02.756253Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:00.845187Z","time spent":"1.911054399s","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Range","request count":0,"request size":99,"response count":1,"response size":10400,"request content":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/ciliuml2announcementpolicies.cilium.io\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:02.756256Z","caller":"traceutil/trace.go:172","msg":"trace[1858035689] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:1106; }","duration":"1.446203458s","start":"2026-03-30T21:31:01.310037Z","end":"2026-03-30T21:31:02.756240Z","steps":["trace[1858035689] 'agreement among raft nodes before linearized reading' (duration: 1.445825829s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.756356Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:01.310014Z","time spent":"1.446325321s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":62,"response count":1,"response size":518,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 "} 2026/03/30 21:31:02 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T21:31:02.822006Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360386,"received-request-id":16727104867003360382} {"level":"info","ts":"2026-03-30T21:31:02.850626Z","caller":"traceutil/trace.go:172","msg":"trace[895815237] transaction","detail":"{read_only:false; response_revision:1108; number_of_response:1; }","duration":"720.927808ms","start":"2026-03-30T21:31:02.129665Z","end":"2026-03-30T21:31:02.850593Z","steps":["trace[895815237] 'process raft request' (duration: 720.675662ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.850811Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.129632Z","time spent":"721.083132ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:02.850873Z","caller":"traceutil/trace.go:172","msg":"trace[1344340635] transaction","detail":"{read_only:false; response_revision:1109; number_of_response:1; }","duration":"713.44631ms","start":"2026-03-30T21:31:02.137399Z","end":"2026-03-30T21:31:02.850845Z","steps":["trace[1344340635] 'process raft request' (duration: 713.079111ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.851064Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.137368Z","time spent":"713.583753ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7983,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:02.851159Z","caller":"traceutil/trace.go:172","msg":"trace[1594175636] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1109; }","duration":"656.651693ms","start":"2026-03-30T21:31:02.194485Z","end":"2026-03-30T21:31:02.851136Z","steps":["trace[1594175636] 'process raft request' (duration: 656.200502ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.851295Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.194455Z","time spent":"656.783127ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":103,"response count":0,"response size":188522,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:02.915850Z","caller":"traceutil/trace.go:172","msg":"trace[2784949] linearizableReadLoop","detail":"{readStateIndex:1331; appliedIndex:1331; }","duration":"159.976537ms","start":"2026-03-30T21:31:02.755850Z","end":"2026-03-30T21:31:02.915827Z","steps":["trace[2784949] 'read index received' (duration: 159.966916ms)","trace[2784949] 'applied index is now lower than readState.Index' (duration: 8.091µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:02.916017Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"445.462721ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/podtemplates\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:02.916048Z","caller":"traceutil/trace.go:172","msg":"trace[525724506] range","detail":"{range_begin:/registry/podtemplates; range_end:; response_count:0; response_revision:1113; }","duration":"445.513482ms","start":"2026-03-30T21:31:02.470525Z","end":"2026-03-30T21:31:02.916039Z","steps":["trace[525724506] 'agreement among raft nodes before linearized reading' (duration: 445.42109ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.916078Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.470502Z","time spent":"445.567234ms","remote":"127.0.0.1:41366","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/podtemplates\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:02.916278Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"786.674821ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:02.916349Z","caller":"traceutil/trace.go:172","msg":"trace[2067586937] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1114; }","duration":"786.749403ms","start":"2026-03-30T21:31:02.129586Z","end":"2026-03-30T21:31:02.916335Z","steps":["trace[2067586937] 'agreement among raft nodes before linearized reading' (duration: 786.652241ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:02.916396Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.129561Z","time spent":"786.827216ms","remote":"127.0.0.1:41158","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:02.916415Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"786.791035ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:02.916447Z","caller":"traceutil/trace.go:172","msg":"trace[1573397059] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1114; }","duration":"786.823106ms","start":"2026-03-30T21:31:02.129615Z","end":"2026-03-30T21:31:02.916438Z","steps":["trace[1573397059] 'agreement among raft nodes before linearized reading' (duration: 786.777105ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:03.312514Z","caller":"traceutil/trace.go:172","msg":"trace[1272768048] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1116; }","duration":"519.22644ms","start":"2026-03-30T21:31:02.793267Z","end":"2026-03-30T21:31:03.312493Z","steps":["trace[1272768048] 'process raft request' (duration: 519.1909ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:03.312666Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.793236Z","time spent":"519.365014ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":62,"response count":0,"response size":529,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:03.416222Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360390,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:03.425826Z","caller":"traceutil/trace.go:172","msg":"trace[274649549] linearizableReadLoop","detail":"{readStateIndex:1335; appliedIndex:1335; }","duration":"509.875735ms","start":"2026-03-30T21:31:02.915920Z","end":"2026-03-30T21:31:03.425796Z","steps":["trace[274649549] 'read index received' (duration: 509.867385ms)","trace[274649549] 'applied index is now lower than readState.Index' (duration: 6.62µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:03.426105Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"667.39235ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:3 size:387"} {"level":"info","ts":"2026-03-30T21:31:03.426155Z","caller":"traceutil/trace.go:172","msg":"trace[877413964] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:3; response_revision:1116; }","duration":"667.445341ms","start":"2026-03-30T21:31:02.758690Z","end":"2026-03-30T21:31:03.426136Z","steps":["trace[877413964] 'agreement among raft nodes before linearized reading' (duration: 667.274757ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:03.426166Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"568.96563ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a1bbd58260c712\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"warn","ts":"2026-03-30T21:31:03.426042Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"566.904537ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-qzq6k\" limit:1 ","response":"range_response_count:1 size:18521"} {"level":"info","ts":"2026-03-30T21:31:03.426227Z","caller":"traceutil/trace.go:172","msg":"trace[506773749] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18a1bbd58260c712; range_end:; response_count:1; response_revision:1116; }","duration":"569.002631ms","start":"2026-03-30T21:31:02.857187Z","end":"2026-03-30T21:31:03.426190Z","steps":["trace[506773749] 'agreement among raft nodes before linearized reading' (duration: 568.659482ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:03.426610Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.857162Z","time spent":"569.433572ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":817,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a1bbd58260c712\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:03.426241Z","caller":"traceutil/trace.go:172","msg":"trace[349312027] range","detail":"{range_begin:/registry/pods/kube-system/cilium-qzq6k; range_end:; response_count:1; response_revision:1116; }","duration":"567.115842ms","start":"2026-03-30T21:31:02.859104Z","end":"2026-03-30T21:31:03.426219Z","steps":["trace[349312027] 'agreement among raft nodes before linearized reading' (duration: 566.738453ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:03.426188Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.758640Z","time spent":"667.539843ms","remote":"127.0.0.1:41216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":3,"response size":410,"request content":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" "} {"level":"warn","ts":"2026-03-30T21:31:03.427251Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.859088Z","time spent":"568.041676ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18544,"request content":"key:\"/registry/pods/kube-system/cilium-qzq6k\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:03.426369Z","caller":"traceutil/trace.go:172","msg":"trace[509662376] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1116; }","duration":"613.61407ms","start":"2026-03-30T21:31:02.812742Z","end":"2026-03-30T21:31:03.426356Z","steps":["trace[509662376] 'process raft request' (duration: 613.290882ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:03.427501Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.812713Z","time spent":"614.721628ms","remote":"127.0.0.1:41248","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":103,"response count":0,"response size":188531,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:03.926328Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360391,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:31:04.302073Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360391,"received-request-id":16727104867003360390} {"level":"info","ts":"2026-03-30T21:31:04.373456Z","caller":"traceutil/trace.go:172","msg":"trace[1053746780] linearizableReadLoop","detail":"{readStateIndex:1343; appliedIndex:1343; }","duration":"947.52274ms","start":"2026-03-30T21:31:03.425915Z","end":"2026-03-30T21:31:04.373437Z","steps":["trace[1053746780] 'read index received' (duration: 947.51639ms)","trace[1053746780] 'applied index is now lower than readState.Index' (duration: 5.19µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:04.389788Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.466169576s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:04.389849Z","caller":"traceutil/trace.go:172","msg":"trace[251183267] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1122; }","duration":"1.466248158s","start":"2026-03-30T21:31:02.923587Z","end":"2026-03-30T21:31:04.389836Z","steps":["trace[251183267] 'agreement among raft nodes before linearized reading' (duration: 1.450031448s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.389886Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:02.923571Z","time spent":"1.466306909s","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:04.389957Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"966.912351ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:31:04.390047Z","caller":"traceutil/trace.go:172","msg":"trace[1957719036] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1122; }","duration":"967.012283ms","start":"2026-03-30T21:31:03.423011Z","end":"2026-03-30T21:31:04.390023Z","steps":["trace[1957719036] 'agreement among raft nodes before linearized reading' (duration: 950.591128ms)","trace[1957719036] 'range keys from in-memory index tree' (duration: 16.20467ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:04.390106Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:03.422994Z","time spent":"967.086824ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:04.390284Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.473093212s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:04.390340Z","caller":"traceutil/trace.go:172","msg":"trace[824641177] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1122; }","duration":"1.473148444s","start":"2026-03-30T21:31:02.917179Z","end":"2026-03-30T21:31:04.390328Z","steps":["trace[824641177] 'agreement among raft nodes before linearized reading' (duration: 1.456446611s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:04.392229Z","caller":"traceutil/trace.go:172","msg":"trace[1611296766] transaction","detail":"{read_only:false; response_revision:1129; number_of_response:1; }","duration":"962.902839ms","start":"2026-03-30T21:31:03.429312Z","end":"2026-03-30T21:31:04.392215Z","steps":["trace[1611296766] 'process raft request' (duration: 962.862728ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.393060Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:03.429289Z","time spent":"962.988981ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:04.393334Z","caller":"traceutil/trace.go:172","msg":"trace[765126038] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1129; }","duration":"963.35734ms","start":"2026-03-30T21:31:03.429963Z","end":"2026-03-30T21:31:04.393320Z","steps":["trace[765126038] 'process raft request' (duration: 963.150765ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:04.393363Z","caller":"traceutil/trace.go:172","msg":"trace[283724967] transaction","detail":"{read_only:false; response_revision:1130; number_of_response:1; }","duration":"952.087125ms","start":"2026-03-30T21:31:03.441223Z","end":"2026-03-30T21:31:04.393310Z","steps":["trace[283724967] 'process raft request' (duration: 951.983492ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.393402Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:03.429952Z","time spent":"963.419402ms","remote":"127.0.0.1:41448","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":51,"response count":0,"response size":494,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:04.393450Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:03.441200Z","time spent":"952.206739ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18604,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:04.419225Z","caller":"traceutil/trace.go:172","msg":"trace[305697381] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1131; }","duration":"427.275921ms","start":"2026-03-30T21:31:03.991937Z","end":"2026-03-30T21:31:04.419213Z","steps":["trace[305697381] 'process raft request' (duration: 427.163998ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.419329Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:03.991905Z","time spent":"427.369803ms","remote":"127.0.0.1:41456","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":34,"response count":0,"response size":4135,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:04.456415Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"284.261159ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T21:31:04.456478Z","caller":"traceutil/trace.go:172","msg":"trace[1018563503] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1132; }","duration":"284.3304ms","start":"2026-03-30T21:31:04.172134Z","end":"2026-03-30T21:31:04.456464Z","steps":["trace[1018563503] 'agreement among raft nodes before linearized reading' (duration: 284.208927ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.456501Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"107.548893ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:31:04.456570Z","caller":"traceutil/trace.go:172","msg":"trace[643758860] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1132; }","duration":"107.634436ms","start":"2026-03-30T21:31:04.348924Z","end":"2026-03-30T21:31:04.456558Z","steps":["trace[643758860] 'agreement among raft nodes before linearized reading' (duration: 107.41343ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.456532Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"587.701193ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T21:31:04.456664Z","caller":"traceutil/trace.go:172","msg":"trace[1419622185] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1132; }","duration":"587.822646ms","start":"2026-03-30T21:31:03.868789Z","end":"2026-03-30T21:31:04.456612Z","steps":["trace[1419622185] 'agreement among raft nodes before linearized reading' (duration: 587.563779ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.456753Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:03.868769Z","time spent":"587.971779ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:04.509657Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"118.567983ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:04.509747Z","caller":"traceutil/trace.go:172","msg":"trace[256250048] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1141; }","duration":"118.667555ms","start":"2026-03-30T21:31:04.391063Z","end":"2026-03-30T21:31:04.509730Z","steps":["trace[256250048] 'agreement among raft nodes before linearized reading' (duration: 118.543272ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.510113Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"109.884523ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 ","response":"range_response_count:1 size:7998"} {"level":"info","ts":"2026-03-30T21:31:04.510160Z","caller":"traceutil/trace.go:172","msg":"trace[1262804802] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-2; range_end:; response_count:1; response_revision:1141; }","duration":"109.933815ms","start":"2026-03-30T21:31:04.400214Z","end":"2026-03-30T21:31:04.510148Z","steps":["trace[1262804802] 'agreement among raft nodes before linearized reading' (duration: 109.76678ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.510373Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"118.487631ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:04.510430Z","caller":"traceutil/trace.go:172","msg":"trace[1288859577] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1141; }","duration":"118.545262ms","start":"2026-03-30T21:31:04.391873Z","end":"2026-03-30T21:31:04.510418Z","steps":["trace[1288859577] 'agreement among raft nodes before linearized reading' (duration: 118.46978ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.510751Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"114.711975ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.78\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-30T21:31:04.510801Z","caller":"traceutil/trace.go:172","msg":"trace[308202109] range","detail":"{range_begin:/registry/masterleases/199.204.45.78; range_end:; response_count:1; response_revision:1141; }","duration":"114.762887ms","start":"2026-03-30T21:31:04.396025Z","end":"2026-03-30T21:31:04.510788Z","steps":["trace[308202109] 'agreement among raft nodes before linearized reading' (duration: 114.642024ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:04.654851Z","caller":"traceutil/trace.go:172","msg":"trace[1559064642] linearizableReadLoop","detail":"{readStateIndex:1367; appliedIndex:1367; }","duration":"141.628438ms","start":"2026-03-30T21:31:04.513202Z","end":"2026-03-30T21:31:04.654831Z","steps":["trace[1559064642] 'read index received' (duration: 141.621278ms)","trace[1559064642] 'applied index is now lower than readState.Index' (duration: 6.08µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:04.696789Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"183.531289ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:04.696865Z","caller":"traceutil/trace.go:172","msg":"trace[1548475919] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1142; }","duration":"183.653052ms","start":"2026-03-30T21:31:04.513194Z","end":"2026-03-30T21:31:04.696847Z","steps":["trace[1548475919] 'agreement among raft nodes before linearized reading' (duration: 141.767611ms)","trace[1548475919] 'range keys from in-memory index tree' (duration: 41.742657ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:31:04.697115Z","caller":"traceutil/trace.go:172","msg":"trace[1293092740] transaction","detail":"{read_only:false; response_revision:1144; number_of_response:1; }","duration":"177.616048ms","start":"2026-03-30T21:31:04.519479Z","end":"2026-03-30T21:31:04.697095Z","steps":["trace[1293092740] 'process raft request' (duration: 177.546996ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:04.748610Z","caller":"traceutil/trace.go:172","msg":"trace[122269155] transaction","detail":"{read_only:false; response_revision:1146; number_of_response:1; }","duration":"213.747524ms","start":"2026-03-30T21:31:04.534841Z","end":"2026-03-30T21:31:04.748588Z","steps":["trace[122269155] 'process raft request' (duration: 213.638981ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:04.749798Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"231.714219ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:04.749956Z","caller":"traceutil/trace.go:172","msg":"trace[1026515140] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1146; }","duration":"231.875983ms","start":"2026-03-30T21:31:04.518060Z","end":"2026-03-30T21:31:04.749936Z","steps":["trace[1026515140] 'agreement among raft nodes before linearized reading' (duration: 231.102964ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:04.913426Z","caller":"traceutil/trace.go:172","msg":"trace[576236470] transaction","detail":"{read_only:false; response_revision:1149; number_of_response:1; }","duration":"148.329817ms","start":"2026-03-30T21:31:04.765064Z","end":"2026-03-30T21:31:04.913394Z","steps":["trace[576236470] 'process raft request' (duration: 147.947378ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:05.736977Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"271.815269ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:05.737110Z","caller":"traceutil/trace.go:172","msg":"trace[384306503] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1150; }","duration":"271.978123ms","start":"2026-03-30T21:31:05.465106Z","end":"2026-03-30T21:31:05.737084Z","steps":["trace[384306503] 'range keys from in-memory index tree' (duration: 271.720996ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:05.737219Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"467.491658ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:31:05.737543Z","caller":"traceutil/trace.go:172","msg":"trace[743519481] transaction","detail":"{read_only:false; response_revision:1152; number_of_response:1; }","duration":"421.992224ms","start":"2026-03-30T21:31:05.315522Z","end":"2026-03-30T21:31:05.737514Z","steps":["trace[743519481] 'process raft request' (duration: 421.886781ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:05.737629Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:05.315501Z","time spent":"422.092737ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":762,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:31:05.737527Z","caller":"traceutil/trace.go:172","msg":"trace[1184415477] linearizableReadLoop","detail":"{readStateIndex:1377; appliedIndex:1376; }","duration":"433.70505ms","start":"2026-03-30T21:31:05.303793Z","end":"2026-03-30T21:31:05.737498Z","steps":["trace[1184415477] 'read index received' (duration: 1.222141ms)","trace[1184415477] 'applied index is now lower than readState.Index' (duration: 432.480549ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:05.738025Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"434.226904ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-qzq6k\" limit:1 ","response":"range_response_count:1 size:18792"} {"level":"info","ts":"2026-03-30T21:31:05.738095Z","caller":"traceutil/trace.go:172","msg":"trace[1681246138] range","detail":"{range_begin:/registry/pods/kube-system/cilium-qzq6k; range_end:; response_count:1; response_revision:1152; }","duration":"434.295836ms","start":"2026-03-30T21:31:05.303778Z","end":"2026-03-30T21:31:05.738074Z","steps":["trace[1681246138] 'agreement among raft nodes before linearized reading' (duration: 433.982528ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:05.738144Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:05.303754Z","time spent":"434.377198ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18815,"request content":"key:\"/registry/pods/kube-system/cilium-qzq6k\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:06.428296Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360416,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:06.603383Z","caller":"traceutil/trace.go:172","msg":"trace[1491850238] transaction","detail":"{read_only:false; response_revision:1153; number_of_response:1; }","duration":"849.899566ms","start":"2026-03-30T21:31:05.753459Z","end":"2026-03-30T21:31:06.603358Z","steps":["trace[1491850238] 'process raft request' (duration: 849.458484ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:06.603588Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:05.753426Z","time spent":"850.078891ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18870,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:06.928837Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360416,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:06.941941Z","caller":"traceutil/trace.go:172","msg":"trace[1697899269] linearizableReadLoop","detail":"{readStateIndex:1379; appliedIndex:1381; }","duration":"1.014568363s","start":"2026-03-30T21:31:05.927344Z","end":"2026-03-30T21:31:06.941912Z","steps":["trace[1697899269] 'read index received' (duration: 1.014559693s)","trace[1697899269] 'applied index is now lower than readState.Index' (duration: 7.15µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:07.097018Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.169655867s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:07.097105Z","caller":"traceutil/trace.go:172","msg":"trace[1602650907] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1155; }","duration":"1.169745859s","start":"2026-03-30T21:31:05.927339Z","end":"2026-03-30T21:31:07.097084Z","steps":["trace[1602650907] 'agreement among raft nodes before linearized reading' (duration: 1.014692856s)","trace[1602650907] 'range keys from in-memory index tree' (duration: 154.93623ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:07.097146Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:05.927316Z","time spent":"1.169821191s","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:07.097198Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"154.9244ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-30T21:31:07.189239Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360418,"received-request-id":16727104867003360416} {"level":"info","ts":"2026-03-30T21:31:07.395191Z","caller":"traceutil/trace.go:172","msg":"trace[1411277262] linearizableReadLoop","detail":"{readStateIndex:1383; appliedIndex:1384; }","duration":"453.153345ms","start":"2026-03-30T21:31:06.942016Z","end":"2026-03-30T21:31:07.395169Z","steps":["trace[1411277262] 'read index received' (duration: 453.147115ms)","trace[1411277262] 'applied index is now lower than readState.Index' (duration: 5.19µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:07.395325Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.141499798s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaimtemplates\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:07.395346Z","caller":"traceutil/trace.go:172","msg":"trace[18214509] range","detail":"{range_begin:/registry/resourceclaimtemplates; range_end:; response_count:0; response_revision:1158; }","duration":"1.141542949s","start":"2026-03-30T21:31:06.253797Z","end":"2026-03-30T21:31:07.395340Z","steps":["trace[18214509] 'agreement among raft nodes before linearized reading' (duration: 1.141467647s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:07.395334Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"929.365001ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:07.395375Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"833.258352ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T21:31:07.395412Z","caller":"traceutil/trace.go:172","msg":"trace[374089808] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1158; }","duration":"929.463513ms","start":"2026-03-30T21:31:06.465931Z","end":"2026-03-30T21:31:07.395394Z","steps":["trace[374089808] 'agreement among raft nodes before linearized reading' (duration: 929.323289ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:07.395452Z","caller":"traceutil/trace.go:172","msg":"trace[1663530960] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1158; }","duration":"833.360455ms","start":"2026-03-30T21:31:06.562069Z","end":"2026-03-30T21:31:07.395430Z","steps":["trace[1663530960] 'agreement among raft nodes before linearized reading' (duration: 833.132449ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:07.395366Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:06.253768Z","time spent":"1.141593369s","remote":"127.0.0.1:42230","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":28,"request content":"key:\"/registry/resourceclaimtemplates\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:07.395493Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:06.562045Z","time spent":"833.438877ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:07.396812Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"298.444462ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:07.396831Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"116.927156ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:07.396867Z","caller":"traceutil/trace.go:172","msg":"trace[149390642] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1160; }","duration":"298.498334ms","start":"2026-03-30T21:31:07.098357Z","end":"2026-03-30T21:31:07.396856Z","steps":["trace[149390642] 'agreement among raft nodes before linearized reading' (duration: 298.412321ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:07.396905Z","caller":"traceutil/trace.go:172","msg":"trace[2108427174] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1160; }","duration":"117.008869ms","start":"2026-03-30T21:31:07.279880Z","end":"2026-03-30T21:31:07.396889Z","steps":["trace[2108427174] 'agreement among raft nodes before linearized reading' (duration: 116.888666ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:07.783096Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"184.627834ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/mutatingwebhookconfigurations\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:07.783215Z","caller":"traceutil/trace.go:172","msg":"trace[1497928027] range","detail":"{range_begin:/registry/mutatingwebhookconfigurations; range_end:; response_count:0; response_revision:1160; }","duration":"184.752047ms","start":"2026-03-30T21:31:07.598440Z","end":"2026-03-30T21:31:07.783192Z","steps":["trace[1497928027] 'range keys from in-memory index tree' (duration: 183.145107ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:07.783456Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"314.651543ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:07.783508Z","caller":"traceutil/trace.go:172","msg":"trace[91823921] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1160; }","duration":"315.165056ms","start":"2026-03-30T21:31:07.468327Z","end":"2026-03-30T21:31:07.783492Z","steps":["trace[91823921] 'range keys from in-memory index tree' (duration: 314.575241ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:07.855564Z","caller":"traceutil/trace.go:172","msg":"trace[133549144] transaction","detail":"{read_only:false; response_revision:1161; number_of_response:1; }","duration":"155.713241ms","start":"2026-03-30T21:31:07.699821Z","end":"2026-03-30T21:31:07.855534Z","steps":["trace[133549144] 'process raft request' (duration: 155.444703ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:07.856194Z","caller":"traceutil/trace.go:172","msg":"trace[1048540586] linearizableReadLoop","detail":"{readStateIndex:1387; appliedIndex:1387; }","duration":"126.289674ms","start":"2026-03-30T21:31:07.729869Z","end":"2026-03-30T21:31:07.856159Z","steps":["trace[1048540586] 'read index received' (duration: 126.278503ms)","trace[1048540586] 'applied index is now lower than readState.Index' (duration: 9.011µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:07.856499Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"126.630153ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T21:31:07.856557Z","caller":"traceutil/trace.go:172","msg":"trace[1627862438] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1161; }","duration":"126.692164ms","start":"2026-03-30T21:31:07.729852Z","end":"2026-03-30T21:31:07.856544Z","steps":["trace[1627862438] 'agreement among raft nodes before linearized reading' (duration: 126.470469ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:08.980494Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360425,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:31:09.065423Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"249.138095ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:31:09.065968Z","caller":"traceutil/trace.go:172","msg":"trace[1046876938] transaction","detail":"{read_only:false; response_revision:1164; number_of_response:1; }","duration":"588.693945ms","start":"2026-03-30T21:31:08.477259Z","end":"2026-03-30T21:31:09.065953Z","steps":["trace[1046876938] 'process raft request' (duration: 338.958004ms)","trace[1046876938] 'compare' (duration: 249.044413ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:09.066034Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:08.477220Z","time spent":"588.794708ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":730,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:31:09.078886Z","caller":"traceutil/trace.go:172","msg":"trace[2026742733] linearizableReadLoop","detail":"{readStateIndex:1389; appliedIndex:1390; }","duration":"599.237062ms","start":"2026-03-30T21:31:08.479625Z","end":"2026-03-30T21:31:09.078862Z","steps":["trace[2026742733] 'read index received' (duration: 599.229232ms)","trace[2026742733] 'applied index is now lower than readState.Index' (duration: 6.63µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:09.079040Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"599.412346ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csistoragecapacities\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:09.079089Z","caller":"traceutil/trace.go:172","msg":"trace[2076668604] range","detail":"{range_begin:/registry/csistoragecapacities; range_end:; response_count:0; response_revision:1164; }","duration":"599.467888ms","start":"2026-03-30T21:31:08.479606Z","end":"2026-03-30T21:31:09.079074Z","steps":["trace[2076668604] 'agreement among raft nodes before linearized reading' (duration: 599.364705ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:09.079121Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:08.479573Z","time spent":"599.539119ms","remote":"127.0.0.1:41946","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":28,"request content":"key:\"/registry/csistoragecapacities\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:09.126545Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360427,"received-request-id":16727104867003360425} {"level":"warn","ts":"2026-03-30T21:31:09.126749Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"616.943371ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:09.126799Z","caller":"traceutil/trace.go:172","msg":"trace[1267503715] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1168; }","duration":"617.014673ms","start":"2026-03-30T21:31:08.509764Z","end":"2026-03-30T21:31:09.126779Z","steps":["trace[1267503715] 'agreement among raft nodes before linearized reading' (duration: 616.90087ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:09.126787Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"199.68857ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:09.126797Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"566.400238ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T21:31:09.127033Z","caller":"traceutil/trace.go:172","msg":"trace[942136343] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1168; }","duration":"566.573152ms","start":"2026-03-30T21:31:08.560379Z","end":"2026-03-30T21:31:09.126952Z","steps":["trace[942136343] 'agreement among raft nodes before linearized reading' (duration: 566.354497ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:09.127276Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:08.560346Z","time spent":"566.910581ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:09.126873Z","caller":"traceutil/trace.go:172","msg":"trace[633897766] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1168; }","duration":"199.786243ms","start":"2026-03-30T21:31:08.927067Z","end":"2026-03-30T21:31:09.126853Z","steps":["trace[633897766] 'agreement among raft nodes before linearized reading' (duration: 199.645689ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:09.952053Z","caller":"traceutil/trace.go:172","msg":"trace[313814658] transaction","detail":"{read_only:false; response_revision:1177; number_of_response:1; }","duration":"348.001558ms","start":"2026-03-30T21:31:09.604030Z","end":"2026-03-30T21:31:09.952032Z","steps":["trace[313814658] 'process raft request' (duration: 347.756732ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:09.952217Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:09.604004Z","time spent":"348.127511ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":19125,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:09.973480Z","caller":"traceutil/trace.go:172","msg":"trace[44258010] linearizableReadLoop","detail":"{readStateIndex:1402; appliedIndex:1403; }","duration":"110.791424ms","start":"2026-03-30T21:31:09.862659Z","end":"2026-03-30T21:31:09.973450Z","steps":["trace[44258010] 'read index received' (duration: 110.780664ms)","trace[44258010] 'applied index is now lower than readState.Index' (duration: 9.22µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:09.973788Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"111.092841ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:31:09.973904Z","caller":"traceutil/trace.go:172","msg":"trace[1717409522] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1177; }","duration":"111.234635ms","start":"2026-03-30T21:31:09.862647Z","end":"2026-03-30T21:31:09.973881Z","steps":["trace[1717409522] 'agreement among raft nodes before linearized reading' (duration: 110.892036ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:10.628382Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"441.675942ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-30T21:31:10.628598Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"163.630828ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:10.628643Z","caller":"traceutil/trace.go:172","msg":"trace[540828119] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1177; }","duration":"163.676379ms","start":"2026-03-30T21:31:10.464950Z","end":"2026-03-30T21:31:10.628627Z","steps":["trace[540828119] 'range keys from in-memory index tree' (duration: 163.565665ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:10.629305Z","caller":"traceutil/trace.go:172","msg":"trace[238937321] linearizableReadLoop","detail":"{readStateIndex:1405; appliedIndex:1403; }","duration":"655.70337ms","start":"2026-03-30T21:31:09.973547Z","end":"2026-03-30T21:31:10.629250Z","steps":["trace[238937321] 'read index received' (duration: 215.839174ms)","trace[238937321] 'applied index is now lower than readState.Index' (duration: 439.862486ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:10.748716Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"821.47046ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:10.748809Z","caller":"traceutil/trace.go:172","msg":"trace[1904192417] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1178; }","duration":"821.573444ms","start":"2026-03-30T21:31:09.927215Z","end":"2026-03-30T21:31:10.748788Z","steps":["trace[1904192417] 'agreement among raft nodes before linearized reading' (duration: 702.1225ms)","trace[1904192417] 'range keys from in-memory index tree' (duration: 119.313891ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:10.748859Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:09.927192Z","time spent":"821.652825ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:11.129972Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360436,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:11.453841Z","caller":"traceutil/trace.go:172","msg":"trace[1774770761] linearizableReadLoop","detail":"{readStateIndex:1405; appliedIndex:1407; }","duration":"824.453643ms","start":"2026-03-30T21:31:10.629352Z","end":"2026-03-30T21:31:11.453805Z","steps":["trace[1774770761] 'read index received' (duration: 824.436522ms)","trace[1774770761] 'applied index is now lower than readState.Index' (duration: 11.981µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:11.453909Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"825.186502ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:11.453992Z","caller":"traceutil/trace.go:172","msg":"trace[1004456891] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1180; }","duration":"825.280355ms","start":"2026-03-30T21:31:10.628689Z","end":"2026-03-30T21:31:11.453969Z","steps":["trace[1004456891] 'agreement among raft nodes before linearized reading' (duration: 825.142821ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:11.454031Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.031445173s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:11.454113Z","caller":"traceutil/trace.go:172","msg":"trace[1570063095] range","detail":"{range_begin:/registry/endpointslices; range_end:; response_count:0; response_revision:1180; }","duration":"1.031544305s","start":"2026-03-30T21:31:10.422550Z","end":"2026-03-30T21:31:11.454094Z","steps":["trace[1570063095] 'agreement among raft nodes before linearized reading' (duration: 1.03136312s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:11.454158Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:10.422532Z","time spent":"1.031619097s","remote":"127.0.0.1:41658","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/endpointslices\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:11.573626Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360440,"received-request-id":16727104867003360436} {"level":"info","ts":"2026-03-30T21:31:11.698511Z","caller":"traceutil/trace.go:172","msg":"trace[1850250374] transaction","detail":"{read_only:false; response_revision:1182; number_of_response:1; }","duration":"578.412724ms","start":"2026-03-30T21:31:11.120078Z","end":"2026-03-30T21:31:11.698490Z","steps":["trace[1850250374] 'process raft request' (duration: 578.320551ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:11.698697Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:11.120050Z","time spent":"578.581107ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":676,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:11.742935Z","caller":"traceutil/trace.go:172","msg":"trace[645665570] linearizableReadLoop","detail":"{readStateIndex:1407; appliedIndex:1410; }","duration":"289.002376ms","start":"2026-03-30T21:31:11.453917Z","end":"2026-03-30T21:31:11.742919Z","steps":["trace[645665570] 'read index received' (duration: 288.996126ms)","trace[645665570] 'applied index is now lower than readState.Index' (duration: 5.12µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:11.842263Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.092094349s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:11.842375Z","caller":"traceutil/trace.go:172","msg":"trace[970502222] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1183; }","duration":"1.092213512s","start":"2026-03-30T21:31:10.750141Z","end":"2026-03-30T21:31:11.842355Z","steps":["trace[970502222] 'agreement among raft nodes before linearized reading' (duration: 992.854915ms)","trace[970502222] 'range keys from in-memory index tree' (duration: 99.189372ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:11.842375Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"550.903226ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-03-30T21:31:11.842418Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:10.750124Z","time spent":"1.092284254s","remote":"127.0.0.1:41158","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:31:11.842440Z","caller":"traceutil/trace.go:172","msg":"trace[161149155] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1183; }","duration":"550.986008ms","start":"2026-03-30T21:31:11.291437Z","end":"2026-03-30T21:31:11.842423Z","steps":["trace[161149155] 'agreement among raft nodes before linearized reading' (duration: 451.486978ms)","trace[161149155] 'range keys from in-memory index tree' (duration: 99.284225ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:11.842474Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:11.291412Z","time spent":"551.05471ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:11.842700Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.088398785s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingress\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:11.842736Z","caller":"traceutil/trace.go:172","msg":"trace[1913250748] range","detail":"{range_begin:/registry/ingress; range_end:; response_count:0; response_revision:1183; }","duration":"1.088427956s","start":"2026-03-30T21:31:10.754292Z","end":"2026-03-30T21:31:11.842720Z","steps":["trace[1913250748] 'agreement among raft nodes before linearized reading' (duration: 988.644879ms)","trace[1913250748] 'range keys from in-memory index tree' (duration: 99.727296ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:11.842759Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:10.754281Z","time spent":"1.088472508s","remote":"127.0.0.1:41704","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":28,"request content":"key:\"/registry/ingress\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:11.843056Z","caller":"traceutil/trace.go:172","msg":"trace[568738992] linearizableReadLoop","detail":"{readStateIndex:1413; appliedIndex:1410; }","duration":"100.054124ms","start":"2026-03-30T21:31:11.742991Z","end":"2026-03-30T21:31:11.843045Z","steps":["trace[568738992] 'read index received' (duration: 1.93592ms)","trace[568738992] 'applied index is now lower than readState.Index' (duration: 98.116804ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:11.843109Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"377.647391ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:11.843127Z","caller":"traceutil/trace.go:172","msg":"trace[722078654] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1186; }","duration":"377.667751ms","start":"2026-03-30T21:31:11.465453Z","end":"2026-03-30T21:31:11.843121Z","steps":["trace[722078654] 'agreement among raft nodes before linearized reading' (duration: 377.63456ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:11.843135Z","caller":"traceutil/trace.go:172","msg":"trace[143420843] transaction","detail":"{read_only:false; response_revision:1185; number_of_response:1; }","duration":"659.939837ms","start":"2026-03-30T21:31:11.183176Z","end":"2026-03-30T21:31:11.843116Z","steps":["trace[143420843] 'process raft request' (duration: 659.610849ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:11.843209Z","caller":"traceutil/trace.go:172","msg":"trace[1264356898] transaction","detail":"{read_only:false; response_revision:1186; number_of_response:1; }","duration":"505.602025ms","start":"2026-03-30T21:31:11.337576Z","end":"2026-03-30T21:31:11.843178Z","steps":["trace[1264356898] 'process raft request' (duration: 505.240465ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:11.843246Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:11.183143Z","time spent":"660.033229ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:11.843324Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:11.337549Z","time spent":"505.726027ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":709,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:31:12.353016Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360442,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:12.538626Z","caller":"traceutil/trace.go:172","msg":"trace[1691515536] transaction","detail":"{read_only:false; response_revision:1187; number_of_response:1; }","duration":"691.562648ms","start":"2026-03-30T21:31:11.847044Z","end":"2026-03-30T21:31:12.538606Z","steps":["trace[1691515536] 'process raft request' (duration: 691.383963ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:12.538774Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:11.847028Z","time spent":"691.67007ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":708,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:31:12.540655Z","caller":"traceutil/trace.go:172","msg":"trace[1910017059] linearizableReadLoop","detail":"{readStateIndex:1414; appliedIndex:1416; }","duration":"688.48788ms","start":"2026-03-30T21:31:11.852152Z","end":"2026-03-30T21:31:12.540640Z","steps":["trace[1910017059] 'read index received' (duration: 688.48384ms)","trace[1910017059] 'applied index is now lower than readState.Index' (duration: 3.32µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:12.540792Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"688.622683ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:12.540953Z","caller":"traceutil/trace.go:172","msg":"trace[1089082052] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1189; }","duration":"688.788627ms","start":"2026-03-30T21:31:11.852145Z","end":"2026-03-30T21:31:12.540933Z","steps":["trace[1089082052] 'agreement among raft nodes before linearized reading' (duration: 688.571072ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:12.541220Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:11.852136Z","time spent":"688.91427ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:12.542199Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"410.592803ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-30T21:31:12.542262Z","caller":"traceutil/trace.go:172","msg":"trace[58422068] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1189; }","duration":"410.662455ms","start":"2026-03-30T21:31:12.131586Z","end":"2026-03-30T21:31:12.542249Z","steps":["trace[58422068] 'agreement among raft nodes before linearized reading' (duration: 410.469689ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:12.542299Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:12.131561Z","time spent":"410.729707ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:13.163637Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360446,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:31:13.663859Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360446,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:14.027940Z","caller":"traceutil/trace.go:172","msg":"trace[1394499676] linearizableReadLoop","detail":"{readStateIndex:1416; appliedIndex:1417; }","duration":"1.365266268s","start":"2026-03-30T21:31:12.662640Z","end":"2026-03-30T21:31:14.027906Z","steps":["trace[1394499676] 'read index received' (duration: 1.365252928s)","trace[1394499676] 'applied index is now lower than readState.Index' (duration: 11.64µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:14.077059Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.414398639s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-qzq6k\" limit:1 ","response":"range_response_count:1 size:19140"} {"level":"info","ts":"2026-03-30T21:31:14.077150Z","caller":"traceutil/trace.go:172","msg":"trace[405361310] range","detail":"{range_begin:/registry/pods/kube-system/cilium-qzq6k; range_end:; response_count:1; response_revision:1190; }","duration":"1.414496352s","start":"2026-03-30T21:31:12.662630Z","end":"2026-03-30T21:31:14.077126Z","steps":["trace[405361310] 'agreement among raft nodes before linearized reading' (duration: 1.365384661s)","trace[405361310] 'range keys from in-memory index tree' (duration: 48.874234ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:14.077196Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:12.662612Z","time spent":"1.414571123s","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":19163,"request content":"key:\"/registry/pods/kube-system/cilium-qzq6k\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:14.077385Z","caller":"traceutil/trace.go:172","msg":"trace[1228661534] transaction","detail":"{read_only:false; response_revision:1191; number_of_response:1; }","duration":"1.413796033s","start":"2026-03-30T21:31:12.663556Z","end":"2026-03-30T21:31:14.077352Z","steps":["trace[1228661534] 'process raft request' (duration: 1.364529899s)","trace[1228661534] 'compare' (duration: 49.026788ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:14.077536Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:12.663528Z","time spent":"1.413948137s","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":757,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:31:14.251757Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"127.581438ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T21:31:14.395793Z","caller":"traceutil/trace.go:172","msg":"trace[523369510] linearizableReadLoop","detail":"{readStateIndex:1421; appliedIndex:1421; }","duration":"367.724411ms","start":"2026-03-30T21:31:14.028039Z","end":"2026-03-30T21:31:14.395763Z","steps":["trace[523369510] 'read index received' (duration: 367.714161ms)","trace[523369510] 'applied index is now lower than readState.Index' (duration: 9.07µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:14.396204Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"930.613916ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:14.396269Z","caller":"traceutil/trace.go:172","msg":"trace[605317888] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1193; }","duration":"930.895193ms","start":"2026-03-30T21:31:13.465355Z","end":"2026-03-30T21:31:14.396250Z","steps":["trace[605317888] 'agreement among raft nodes before linearized reading' (duration: 930.578285ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:14.396263Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.470317148s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:14.396348Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.019200209s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T21:31:14.396361Z","caller":"traceutil/trace.go:172","msg":"trace[670343264] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1193; }","duration":"1.470432601s","start":"2026-03-30T21:31:12.925905Z","end":"2026-03-30T21:31:14.396338Z","steps":["trace[670343264] 'agreement among raft nodes before linearized reading' (duration: 1.470085882s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:14.396400Z","caller":"traceutil/trace.go:172","msg":"trace[311541796] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1193; }","duration":"1.01925686s","start":"2026-03-30T21:31:13.377130Z","end":"2026-03-30T21:31:14.396387Z","steps":["trace[311541796] 'agreement among raft nodes before linearized reading' (duration: 1.018783858s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:14.396423Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:12.925888Z","time spent":"1.470524433s","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:14.396436Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:13.377106Z","time spent":"1.019319721s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:14.396291Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.159021326s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T21:31:14.396518Z","caller":"traceutil/trace.go:172","msg":"trace[268904625] transaction","detail":"{read_only:false; response_revision:1194; number_of_response:1; }","duration":"305.147348ms","start":"2026-03-30T21:31:14.091353Z","end":"2026-03-30T21:31:14.396501Z","steps":["trace[268904625] 'process raft request' (duration: 304.130412ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:14.396541Z","caller":"traceutil/trace.go:172","msg":"trace[1360940832] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1193; }","duration":"1.159270712s","start":"2026-03-30T21:31:13.237251Z","end":"2026-03-30T21:31:14.396521Z","steps":["trace[1360940832] 'agreement among raft nodes before linearized reading' (duration: 1.158713508s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:14.396593Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:13.237229Z","time spent":"1.159346495s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:14.396642Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:14.091331Z","time spent":"305.240301ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":19391,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:31:14.400187Z","caller":"traceutil/trace.go:172","msg":"trace[714399916] transaction","detail":"{read_only:false; response_revision:1195; number_of_response:1; }","duration":"201.979431ms","start":"2026-03-30T21:31:14.198197Z","end":"2026-03-30T21:31:14.400176Z","steps":["trace[714399916] 'process raft request' (duration: 201.9232ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:14.401769Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"347.728003ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:31:14.402167Z","caller":"traceutil/trace.go:172","msg":"trace[321475862] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1195; }","duration":"348.127363ms","start":"2026-03-30T21:31:14.054015Z","end":"2026-03-30T21:31:14.402143Z","steps":["trace[321475862] 'agreement among raft nodes before linearized reading' (duration: 347.61856ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:14.402274Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:14.053993Z","time spent":"348.262745ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:14.550748Z","caller":"traceutil/trace.go:172","msg":"trace[922477104] linearizableReadLoop","detail":"{readStateIndex:1424; appliedIndex:1424; }","duration":"149.133747ms","start":"2026-03-30T21:31:14.401592Z","end":"2026-03-30T21:31:14.550726Z","steps":["trace[922477104] 'read index received' (duration: 149.126867ms)","trace[922477104] 'applied index is now lower than readState.Index' (duration: 5.82µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:14.906458Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"507.539171ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:14.906506Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"441.305694ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:14.906563Z","caller":"traceutil/trace.go:172","msg":"trace[288810990] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1195; }","duration":"441.369726ms","start":"2026-03-30T21:31:14.465180Z","end":"2026-03-30T21:31:14.906550Z","steps":["trace[288810990] 'range keys from in-memory index tree' (duration: 441.265203ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:14.906529Z","caller":"traceutil/trace.go:172","msg":"trace[18290841] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1195; }","duration":"507.610813ms","start":"2026-03-30T21:31:14.398900Z","end":"2026-03-30T21:31:14.906511Z","steps":["trace[18290841] 'agreement among raft nodes before linearized reading' (duration: 151.95752ms)","trace[18290841] 'range keys from in-memory index tree' (duration: 355.54581ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:14.906611Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:14.398891Z","time spent":"507.713586ms","remote":"127.0.0.1:41158","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:14.906645Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"508.66501ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.78\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-30T21:31:14.906677Z","caller":"traceutil/trace.go:172","msg":"trace[819086925] range","detail":"{range_begin:/registry/masterleases/199.204.45.78; range_end:; response_count:1; response_revision:1195; }","duration":"508.711711ms","start":"2026-03-30T21:31:14.397956Z","end":"2026-03-30T21:31:14.906667Z","steps":["trace[819086925] 'agreement among raft nodes before linearized reading' (duration: 152.898733ms)","trace[819086925] 'range keys from in-memory index tree' (duration: 355.687004ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:14.906704Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:14.397938Z","time spent":"508.757973ms","remote":"127.0.0.1:41216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":1,"response size":156,"request content":"key:\"/registry/masterleases/199.204.45.78\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:15.050919Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360453,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:15.201631Z","caller":"traceutil/trace.go:172","msg":"trace[979946926] linearizableReadLoop","detail":"{readStateIndex:1428; appliedIndex:1431; }","duration":"650.79668ms","start":"2026-03-30T21:31:14.550815Z","end":"2026-03-30T21:31:15.201611Z","steps":["trace[979946926] 'read index received' (duration: 650.791739ms)","trace[979946926] 'applied index is now lower than readState.Index' (duration: 3.921µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:15.201812Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"767.340158ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/servicecidrs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:15.201880Z","caller":"traceutil/trace.go:172","msg":"trace[1833597242] range","detail":"{range_begin:/registry/servicecidrs; range_end:; response_count:0; response_revision:1202; }","duration":"767.4259ms","start":"2026-03-30T21:31:14.434438Z","end":"2026-03-30T21:31:15.201864Z","steps":["trace[1833597242] 'agreement among raft nodes before linearized reading' (duration: 767.298077ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:15.201976Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:14.434416Z","time spent":"767.538952ms","remote":"127.0.0.1:41742","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/servicecidrs\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:15.202628Z","caller":"traceutil/trace.go:172","msg":"trace[1508971698] transaction","detail":"{read_only:false; response_revision:1203; number_of_response:1; }","duration":"307.881991ms","start":"2026-03-30T21:31:14.894724Z","end":"2026-03-30T21:31:15.202606Z","steps":["trace[1508971698] 'process raft request' (duration: 307.108561ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:15.202762Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:14.894703Z","time spent":"308.012363ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":720,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:31:15.333609Z","caller":"traceutil/trace.go:172","msg":"trace[559983531] linearizableReadLoop","detail":"{readStateIndex:1433; appliedIndex:1433; }","duration":"131.877229ms","start":"2026-03-30T21:31:15.201704Z","end":"2026-03-30T21:31:15.333581Z","steps":["trace[559983531] 'read index received' (duration: 131.867629ms)","trace[559983531] 'applied index is now lower than readState.Index' (duration: 8.23µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:15.484035Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"577.427017ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:15.484093Z","caller":"traceutil/trace.go:172","msg":"trace[1227510449] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1203; }","duration":"577.485868ms","start":"2026-03-30T21:31:14.906591Z","end":"2026-03-30T21:31:15.484076Z","steps":["trace[1227510449] 'agreement among raft nodes before linearized reading' (duration: 427.092837ms)","trace[1227510449] 'range keys from in-memory index tree' (duration: 150.323519ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:15.484008Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"570.566782ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:15.484296Z","caller":"traceutil/trace.go:172","msg":"trace[310706562] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1203; }","duration":"570.864199ms","start":"2026-03-30T21:31:14.913421Z","end":"2026-03-30T21:31:15.484285Z","steps":["trace[310706562] 'agreement among raft nodes before linearized reading' (duration: 420.211461ms)","trace[310706562] 'range keys from in-memory index tree' (duration: 150.320959ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:15.484328Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:14.913402Z","time spent":"570.916391ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:31:15.547385Z","caller":"traceutil/trace.go:172","msg":"trace[225844139] transaction","detail":"{read_only:false; response_revision:1204; number_of_response:1; }","duration":"343.736734ms","start":"2026-03-30T21:31:15.203598Z","end":"2026-03-30T21:31:15.547335Z","steps":["trace[225844139] 'process raft request' (duration: 343.404156ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:15.547873Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:15.203576Z","time spent":"344.042753ms","remote":"127.0.0.1:41216","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":118,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:15.717277Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"105.381314ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:31:15.717823Z","caller":"traceutil/trace.go:172","msg":"trace[696107040] linearizableReadLoop","detail":"{readStateIndex:1435; appliedIndex:1434; }","duration":"232.243045ms","start":"2026-03-30T21:31:15.485551Z","end":"2026-03-30T21:31:15.717794Z","steps":["trace[696107040] 'read index received' (duration: 126.806749ms)","trace[696107040] 'applied index is now lower than readState.Index' (duration: 105.250581ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:15.754265Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"268.240242ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:15.754316Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"268.734724ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:15.754391Z","caller":"traceutil/trace.go:172","msg":"trace[321754100] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1205; }","duration":"268.344205ms","start":"2026-03-30T21:31:15.485988Z","end":"2026-03-30T21:31:15.754332Z","steps":["trace[321754100] 'agreement among raft nodes before linearized reading' (duration: 268.189841ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:15.754411Z","caller":"traceutil/trace.go:172","msg":"trace[871729575] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1205; }","duration":"268.844857ms","start":"2026-03-30T21:31:15.485544Z","end":"2026-03-30T21:31:15.754389Z","steps":["trace[871729575] 'agreement among raft nodes before linearized reading' (duration: 232.228945ms)","trace[871729575] 'range keys from in-memory index tree' (duration: 36.474179ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:15.754613Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"204.176361ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:460"} {"level":"info","ts":"2026-03-30T21:31:15.754646Z","caller":"traceutil/trace.go:172","msg":"trace[2009300074] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1205; }","duration":"204.235422ms","start":"2026-03-30T21:31:15.550403Z","end":"2026-03-30T21:31:15.754639Z","steps":["trace[2009300074] 'agreement among raft nodes before linearized reading' (duration: 203.869693ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:16.242417Z","caller":"traceutil/trace.go:172","msg":"trace[102212780] linearizableReadLoop","detail":"{readStateIndex:1437; appliedIndex:1437; }","duration":"319.12794ms","start":"2026-03-30T21:31:15.923261Z","end":"2026-03-30T21:31:16.242389Z","steps":["trace[102212780] 'read index received' (duration: 319.118549ms)","trace[102212780] 'applied index is now lower than readState.Index' (duration: 7.571µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:16.242650Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"319.311785ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:16.242739Z","caller":"traceutil/trace.go:172","msg":"trace[644850984] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1206; }","duration":"319.464489ms","start":"2026-03-30T21:31:15.923252Z","end":"2026-03-30T21:31:16.242716Z","steps":["trace[644850984] 'agreement among raft nodes before linearized reading' (duration: 319.254434ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:16.242790Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:15.923234Z","time spent":"319.54395ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:31:16.361646Z","caller":"traceutil/trace.go:172","msg":"trace[1135921955] transaction","detail":"{read_only:false; response_revision:1207; number_of_response:1; }","duration":"431.751901ms","start":"2026-03-30T21:31:15.929871Z","end":"2026-03-30T21:31:16.361623Z","steps":["trace[1135921955] 'process raft request' (duration: 431.604327ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:16.362724Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:15.929852Z","time spent":"431.877123ms","remote":"127.0.0.1:41492","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":19650,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:31:16.363621Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"433.637198ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-30T21:31:16.363447Z","caller":"traceutil/trace.go:172","msg":"trace[535299614] linearizableReadLoop","detail":"{readStateIndex:1441; appliedIndex:1441; }","duration":"120.944783ms","start":"2026-03-30T21:31:16.242484Z","end":"2026-03-30T21:31:16.363429Z","steps":["trace[535299614] 'read index received' (duration: 120.939273ms)","trace[535299614] 'applied index is now lower than readState.Index' (duration: 4.63µs)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:31:16.363706Z","caller":"traceutil/trace.go:172","msg":"trace[1046380146] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1210; }","duration":"433.69867ms","start":"2026-03-30T21:31:15.929968Z","end":"2026-03-30T21:31:16.363666Z","steps":["trace[1046380146] 'agreement among raft nodes before linearized reading' (duration: 433.596397ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:16.363879Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:15.929953Z","time spent":"433.837914ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:16.364941Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"102.390188ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/daemonsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:16.364974Z","caller":"traceutil/trace.go:172","msg":"trace[424638366] range","detail":"{range_begin:/registry/daemonsets; range_end:; response_count:0; response_revision:1210; }","duration":"102.439249ms","start":"2026-03-30T21:31:16.262528Z","end":"2026-03-30T21:31:16.364967Z","steps":["trace[424638366] 'agreement among raft nodes before linearized reading' (duration: 102.359687ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:16.676582Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"211.741346ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:16.676645Z","caller":"traceutil/trace.go:172","msg":"trace[544309836] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1210; }","duration":"211.819808ms","start":"2026-03-30T21:31:16.464812Z","end":"2026-03-30T21:31:16.676632Z","steps":["trace[544309836] 'range keys from in-memory index tree' (duration: 211.672693ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:17.414038Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360470,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:17.443662Z","caller":"traceutil/trace.go:172","msg":"trace[815328864] linearizableReadLoop","detail":"{readStateIndex:1444; appliedIndex:1444; }","duration":"530.494833ms","start":"2026-03-30T21:31:16.913149Z","end":"2026-03-30T21:31:17.443644Z","steps":["trace[815328864] 'read index received' (duration: 530.489543ms)","trace[815328864] 'applied index is now lower than readState.Index' (duration: 4.2µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:17.444788Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003360471,"received-request-id":16727104867003360470} {"level":"warn","ts":"2026-03-30T21:31:17.450182Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"537.01825ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:17.450255Z","caller":"traceutil/trace.go:172","msg":"trace[392420559] range","detail":"{range_begin:/registry/validatingadmissionpolicybindings; range_end:; response_count:0; response_revision:1213; }","duration":"537.107363ms","start":"2026-03-30T21:31:16.913131Z","end":"2026-03-30T21:31:17.450239Z","steps":["trace[392420559] 'agreement among raft nodes before linearized reading' (duration: 530.692428ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:17.450290Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:16.913105Z","time spent":"537.178464ms","remote":"127.0.0.1:42130","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":28,"request content":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:17.468251Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"540.818527ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:17.468329Z","caller":"traceutil/trace.go:172","msg":"trace[146195586] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1214; }","duration":"540.905059ms","start":"2026-03-30T21:31:16.927409Z","end":"2026-03-30T21:31:17.468314Z","steps":["trace[146195586] 'agreement among raft nodes before linearized reading' (duration: 540.777356ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:17.468367Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:16.927384Z","time spent":"540.973391ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:19.698205Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"146.842386ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:19.698280Z","caller":"traceutil/trace.go:172","msg":"trace[1069493078] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1249; }","duration":"146.937099ms","start":"2026-03-30T21:31:19.551324Z","end":"2026-03-30T21:31:19.698261Z","steps":["trace[1069493078] 'range keys from in-memory index tree' (duration: 146.787035ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:20.114626Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"108.641053ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-30T21:31:20.347837Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"210.550455ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T21:31:20.348194Z","caller":"traceutil/trace.go:172","msg":"trace[1549675403] transaction","detail":"{read_only:false; response_revision:1252; number_of_response:1; }","duration":"517.53731ms","start":"2026-03-30T21:31:19.830636Z","end":"2026-03-30T21:31:20.348174Z","steps":["trace[1549675403] 'process raft request' (duration: 517.369906ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:20.348202Z","caller":"traceutil/trace.go:172","msg":"trace[1943412852] linearizableReadLoop","detail":"{readStateIndex:1487; appliedIndex:1485; }","duration":"560.853974ms","start":"2026-03-30T21:31:19.787315Z","end":"2026-03-30T21:31:20.348169Z","steps":["trace[1943412852] 'read index received' (duration: 331.674134ms)","trace[1943412852] 'applied index is now lower than readState.Index' (duration: 229.17734ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:20.348294Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:19.830608Z","time spent":"517.635102ms","remote":"127.0.0.1:41262","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":788,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:31:20.348511Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"561.183463ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-30T21:31:20.349063Z","caller":"traceutil/trace.go:172","msg":"trace[1179242145] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1252; }","duration":"561.48367ms","start":"2026-03-30T21:31:19.787300Z","end":"2026-03-30T21:31:20.348783Z","steps":["trace[1179242145] 'agreement among raft nodes before linearized reading' (duration: 561.012698ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:20.349156Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:19.787275Z","time spent":"561.862359ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":506,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:20.614085Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"149.448145ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:20.614187Z","caller":"traceutil/trace.go:172","msg":"trace[1370778421] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1252; }","duration":"149.572188ms","start":"2026-03-30T21:31:20.464593Z","end":"2026-03-30T21:31:20.614165Z","steps":["trace[1370778421] 'range keys from in-memory index tree' (duration: 149.345303ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:20.695036Z","caller":"traceutil/trace.go:172","msg":"trace[2070130256] linearizableReadLoop","detail":"{readStateIndex:1491; appliedIndex:1491; }","duration":"346.443884ms","start":"2026-03-30T21:31:20.348568Z","end":"2026-03-30T21:31:20.695012Z","steps":["trace[2070130256] 'read index received' (duration: 346.434694ms)","trace[2070130256] 'applied index is now lower than readState.Index' (duration: 5.6µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:20.695112Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"372.683724ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ipaddresses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:20.695152Z","caller":"traceutil/trace.go:172","msg":"trace[1986550106] range","detail":"{range_begin:/registry/ipaddresses; range_end:; response_count:0; response_revision:1255; }","duration":"372.749175ms","start":"2026-03-30T21:31:20.322394Z","end":"2026-03-30T21:31:20.695143Z","steps":["trace[1986550106] 'agreement among raft nodes before linearized reading' (duration: 372.641862ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:20.695150Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"769.651996ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:20.695177Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:20.322358Z","time spent":"372.810187ms","remote":"127.0.0.1:41730","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/ipaddresses\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:20.695178Z","caller":"traceutil/trace.go:172","msg":"trace[1026036801] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1255; }","duration":"769.683427ms","start":"2026-03-30T21:31:19.925486Z","end":"2026-03-30T21:31:20.695169Z","steps":["trace[1026036801] 'agreement among raft nodes before linearized reading' (duration: 769.629365ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:20.695210Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:19.925471Z","time spent":"769.727208ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:31:20.695394Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"464.832326ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:20.695422Z","caller":"traceutil/trace.go:172","msg":"trace[222931578] range","detail":"{range_begin:/registry/validatingadmissionpolicybindings; range_end:; response_count:0; response_revision:1255; }","duration":"464.861537ms","start":"2026-03-30T21:31:20.230552Z","end":"2026-03-30T21:31:20.695414Z","steps":["trace[222931578] 'agreement among raft nodes before linearized reading' (duration: 464.815356ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:20.695446Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:20.230530Z","time spent":"464.908728ms","remote":"127.0.0.1:42130","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":28,"request content":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:21.073149Z","caller":"traceutil/trace.go:172","msg":"trace[744217982] linearizableReadLoop","detail":"{readStateIndex:1498; appliedIndex:1498; }","duration":"196.69125ms","start":"2026-03-30T21:31:20.876419Z","end":"2026-03-30T21:31:21.073110Z","steps":["trace[744217982] 'read index received' (duration: 196.68561ms)","trace[744217982] 'applied index is now lower than readState.Index' (duration: 4.54µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:21.073386Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"196.936357ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/certificatesigningrequests\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:21.073465Z","caller":"traceutil/trace.go:172","msg":"trace[1430863516] range","detail":"{range_begin:/registry/certificatesigningrequests; range_end:; response_count:0; response_revision:1261; }","duration":"197.035719ms","start":"2026-03-30T21:31:20.876413Z","end":"2026-03-30T21:31:21.073449Z","steps":["trace[1430863516] 'agreement among raft nodes before linearized reading' (duration: 196.812533ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:21.074522Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"148.417409ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:21.074600Z","caller":"traceutil/trace.go:172","msg":"trace[69540893] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1261; }","duration":"148.490871ms","start":"2026-03-30T21:31:20.926091Z","end":"2026-03-30T21:31:21.074582Z","steps":["trace[69540893] 'agreement among raft nodes before linearized reading' (duration: 148.392688ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:40.829063Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41654","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:31:40.847554Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41684","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:31:40.933530Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41702","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:31:40.956265Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41732","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:31:40.982818Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:41736","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:31:41.628561Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37468","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:31:52.309095Z","caller":"traceutil/trace.go:172","msg":"trace[1972223033] transaction","detail":"{read_only:false; response_revision:1473; number_of_response:1; }","duration":"122.518001ms","start":"2026-03-30T21:31:52.186559Z","end":"2026-03-30T21:31:52.309077Z","steps":["trace[1972223033] 'process raft request' (duration: 122.391108ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:52.414237Z","caller":"traceutil/trace.go:172","msg":"trace[1656981940] linearizableReadLoop","detail":"{readStateIndex:1730; appliedIndex:1731; }","duration":"157.902205ms","start":"2026-03-30T21:31:52.256311Z","end":"2026-03-30T21:31:52.414214Z","steps":["trace[1656981940] 'read index received' (duration: 157.895525ms)","trace[1656981940] 'applied index is now lower than readState.Index' (duration: 5.56µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:52.521978Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"265.639975ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:31:52.522063Z","caller":"traceutil/trace.go:172","msg":"trace[924944677] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1473; }","duration":"265.740138ms","start":"2026-03-30T21:31:52.256302Z","end":"2026-03-30T21:31:52.522043Z","steps":["trace[924944677] 'agreement among raft nodes before linearized reading' (duration: 158.06104ms)","trace[924944677] 'range keys from in-memory index tree' (duration: 107.472983ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:52.558579Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"143.639435ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T21:31:52.558702Z","caller":"traceutil/trace.go:172","msg":"trace[131667588] linearizableReadLoop","detail":"{readStateIndex:1732; appliedIndex:1731; }","duration":"144.358374ms","start":"2026-03-30T21:31:52.414324Z","end":"2026-03-30T21:31:52.558682Z","steps":["trace[131667588] 'read index received' (duration: 1.490398ms)","trace[131667588] 'applied index is now lower than readState.Index' (duration: 142.865906ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:31:52.558965Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"231.124853ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T21:31:52.559025Z","caller":"traceutil/trace.go:172","msg":"trace[1974822839] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1473; }","duration":"231.194825ms","start":"2026-03-30T21:31:52.327817Z","end":"2026-03-30T21:31:52.559012Z","steps":["trace[1974822839] 'agreement among raft nodes before linearized reading' (duration: 231.022991ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:31:52.773143Z","caller":"traceutil/trace.go:172","msg":"trace[1722343176] transaction","detail":"{read_only:false; response_revision:1474; number_of_response:1; }","duration":"102.691141ms","start":"2026-03-30T21:31:52.670430Z","end":"2026-03-30T21:31:52.773121Z","steps":["trace[1722343176] 'process raft request' (duration: 102.532336ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:54.813902Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360784,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:31:55.314910Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003360784,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-30T21:31:55.473882Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"45496929f16fe822 is starting a new election at term 2"} {"level":"info","ts":"2026-03-30T21:31:55.473955Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"45496929f16fe822 became pre-candidate at term 2"} {"level":"info","ts":"2026-03-30T21:31:55.473986Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"45496929f16fe822 [logterm: 2, index: 1759] sent MsgPreVote request to 73b132709fee1095 at term 2"} {"level":"info","ts":"2026-03-30T21:31:55.474004Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"45496929f16fe822 [logterm: 2, index: 1759] sent MsgPreVote request to d98d5f2c752cc52d at term 2"} {"level":"info","ts":"2026-03-30T21:31:55.474021Z","logger":"raft","caller":"v3@v3.6.0/node.go:376","msg":"raft.node: 45496929f16fe822 lost leader d98d5f2c752cc52d at term 2"} {"level":"info","ts":"2026-03-30T21:31:55.474096Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"45496929f16fe822 received MsgPreVoteResp from 45496929f16fe822 at term 2"} {"level":"info","ts":"2026-03-30T21:31:55.474200Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"45496929f16fe822 has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T21:31:55.484527Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"45496929f16fe822 received MsgPreVoteResp from 73b132709fee1095 at term 2"} {"level":"info","ts":"2026-03-30T21:31:55.484600Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"45496929f16fe822 has received 2 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T21:31:55.484632Z","logger":"raft","caller":"v3@v3.6.0/raft.go:912","msg":"45496929f16fe822 became candidate at term 3"} {"level":"info","ts":"2026-03-30T21:31:55.484654Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"45496929f16fe822 [logterm: 2, index: 1759] sent MsgVote request to 73b132709fee1095 at term 3"} {"level":"info","ts":"2026-03-30T21:31:55.484670Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"45496929f16fe822 [logterm: 2, index: 1759] sent MsgVote request to d98d5f2c752cc52d at term 3"} {"level":"info","ts":"2026-03-30T21:31:55.508100Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"45496929f16fe822 received MsgVoteResp from 45496929f16fe822 at term 3"} {"level":"info","ts":"2026-03-30T21:31:55.508129Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"45496929f16fe822 has received 1 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T21:31:55.584532Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"45496929f16fe822 received MsgVoteResp from 73b132709fee1095 at term 3"} {"level":"info","ts":"2026-03-30T21:31:55.584600Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"45496929f16fe822 has received 2 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T21:31:55.584638Z","logger":"raft","caller":"v3@v3.6.0/raft.go:970","msg":"45496929f16fe822 became leader at term 3"} {"level":"info","ts":"2026-03-30T21:31:55.584660Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: 45496929f16fe822 elected leader 45496929f16fe822 at term 3"} {"level":"warn","ts":"2026-03-30T21:31:55.584823Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.271426347s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-30T21:31:55.584906Z","caller":"traceutil/trace.go:172","msg":"trace[141027318] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; }","duration":"1.271524069s","start":"2026-03-30T21:31:54.313364Z","end":"2026-03-30T21:31:55.584888Z","steps":["trace[141027318] 'agreement among raft nodes before linearized reading' (duration: 1.271424437s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:55.584965Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:54.313343Z","time spent":"1.271608412s","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":62,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:55.628533Z","caller":"etcdserver/v3_server.go:903","msg":"first commit in current term: resending ReadIndex request"} {"level":"warn","ts":"2026-03-30T21:31:55.629670Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"702.722354ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:31:55.629739Z","caller":"traceutil/trace.go:172","msg":"trace[1831792271] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1499; }","duration":"702.797186ms","start":"2026-03-30T21:31:54.926926Z","end":"2026-03-30T21:31:55.629723Z","steps":["trace[1831792271] 'agreement among raft nodes before linearized reading' (duration: 702.689483ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:55.629745Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.164873216s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:31:55.629777Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:54.926901Z","time spent":"702.867828ms","remote":"127.0.0.1:41126","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:31:55.629791Z","caller":"traceutil/trace.go:172","msg":"trace[1831299318] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1499; }","duration":"1.164920988s","start":"2026-03-30T21:31:54.464858Z","end":"2026-03-30T21:31:55.629779Z","steps":["trace[1831299318] 'agreement among raft nodes before linearized reading' (duration: 1.164838485s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:55.629879Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.228856877s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.78\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"warn","ts":"2026-03-30T21:31:55.629772Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"441.367775ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-30T21:31:55.630146Z","caller":"traceutil/trace.go:172","msg":"trace[601915278] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1499; }","duration":"441.669573ms","start":"2026-03-30T21:31:55.188387Z","end":"2026-03-30T21:31:55.630056Z","steps":["trace[601915278] 'agreement among raft nodes before linearized reading' (duration: 441.243932ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:55.630231Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:55.188369Z","time spent":"441.846097ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"info","ts":"2026-03-30T21:31:55.630294Z","caller":"traceutil/trace.go:172","msg":"trace[124334985] range","detail":"{range_begin:/registry/masterleases/199.204.45.78; range_end:; response_count:1; response_revision:1499; }","duration":"1.228888978s","start":"2026-03-30T21:31:54.401016Z","end":"2026-03-30T21:31:55.629905Z","steps":["trace[124334985] 'agreement among raft nodes before linearized reading' (duration: 1.228797615s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:55.630509Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:31:54.400996Z","time spent":"1.229498525s","remote":"127.0.0.1:41216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":1,"response size":156,"request content":"key:\"/registry/masterleases/199.204.45.78\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:31:55.828545Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"157.407215ms","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:40"} {"level":"warn","ts":"2026-03-30T21:31:55.873351Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"d98d5f2c752cc52d","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"40.46607ms"} {"level":"warn","ts":"2026-03-30T21:31:55.873473Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"73b132709fee1095","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"40.592903ms"} {"level":"info","ts":"2026-03-30T21:31:55.873946Z","caller":"traceutil/trace.go:172","msg":"trace[34677240] transaction","detail":"{read_only:false; response_revision:1500; number_of_response:1; }","duration":"234.156107ms","start":"2026-03-30T21:31:55.639762Z","end":"2026-03-30T21:31:55.873918Z","steps":["trace[34677240] 'process raft request' (duration: 233.963572ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:31:56.028933Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"100.524127ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:31:56.029212Z","caller":"traceutil/trace.go:172","msg":"trace[1542760070] transaction","detail":"{read_only:false; response_revision:1501; number_of_response:1; }","duration":"199.189794ms","start":"2026-03-30T21:31:55.829995Z","end":"2026-03-30T21:31:56.029185Z","steps":["trace[1542760070] 'process raft request' (duration: 98.306869ms)","trace[1542760070] 'get key's previous created_revision and leaseID' {req_type:put; key:/registry/masterleases/199.204.45.78; req_size:116; } (duration: 100.311481ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:32:15.214856Z","caller":"traceutil/trace.go:172","msg":"trace[97110462] transaction","detail":"{read_only:false; response_revision:1618; number_of_response:1; }","duration":"124.355195ms","start":"2026-03-30T21:32:15.090481Z","end":"2026-03-30T21:32:15.214837Z","steps":["trace[97110462] 'process raft request' (duration: 124.220291ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:32:15.655329Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"d98d5f2c752cc52d","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"56.155943ms"} {"level":"warn","ts":"2026-03-30T21:32:15.655406Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"73b132709fee1095","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"56.241365ms"} {"level":"info","ts":"2026-03-30T21:32:15.656585Z","caller":"traceutil/trace.go:172","msg":"trace[1051777056] linearizableReadLoop","detail":"{readStateIndex:1894; appliedIndex:1895; }","duration":"191.638239ms","start":"2026-03-30T21:32:15.464930Z","end":"2026-03-30T21:32:15.656568Z","steps":["trace[1051777056] 'read index received' (duration: 191.634269ms)","trace[1051777056] 'applied index is now lower than readState.Index' (duration: 3.18µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:32:15.656767Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"191.823603ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:32:15.656964Z","caller":"traceutil/trace.go:172","msg":"trace[1491416844] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1621; }","duration":"192.033059ms","start":"2026-03-30T21:32:15.464923Z","end":"2026-03-30T21:32:15.656956Z","steps":["trace[1491416844] 'agreement among raft nodes before linearized reading' (duration: 191.804893ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:32:36.420490Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"d98d5f2c752cc52d","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"146.837747ms"} {"level":"warn","ts":"2026-03-30T21:32:36.420550Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"73b132709fee1095","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"146.901548ms"} {"level":"warn","ts":"2026-03-30T21:32:36.692089Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":16727104867003361062,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:32:36.847007Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"d98d5f2c752cc52d","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"226.434321ms"} {"level":"warn","ts":"2026-03-30T21:32:36.847151Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"73b132709fee1095","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"226.586124ms"} {"level":"info","ts":"2026-03-30T21:32:36.847073Z","caller":"traceutil/trace.go:172","msg":"trace[1268887090] linearizableReadLoop","detail":"{readStateIndex:1991; appliedIndex:1991; }","duration":"655.243356ms","start":"2026-03-30T21:32:36.191799Z","end":"2026-03-30T21:32:36.847042Z","steps":["trace[1268887090] 'read index received' (duration: 655.230956ms)","trace[1268887090] 'applied index is now lower than readState.Index' (duration: 10.46µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:32:36.847600Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"655.747454ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattachments\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:32:36.847795Z","caller":"traceutil/trace.go:172","msg":"trace[81897392] range","detail":"{range_begin:/registry/volumeattachments; range_end:; response_count:0; response_revision:1705; }","duration":"655.964798ms","start":"2026-03-30T21:32:36.191794Z","end":"2026-03-30T21:32:36.847759Z","steps":["trace[81897392] 'agreement among raft nodes before linearized reading' (duration: 655.358328ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:32:36.847874Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:32:36.191773Z","time spent":"656.084281ms","remote":"127.0.0.1:41894","response type":"/etcdserverpb.KV/Range","request count":0,"request size":31,"response count":0,"response size":28,"request content":"key:\"/registry/volumeattachments\" limit:1 "} {"level":"info","ts":"2026-03-30T21:32:36.848197Z","caller":"traceutil/trace.go:172","msg":"trace[1354531051] transaction","detail":"{read_only:false; response_revision:1707; number_of_response:1; }","duration":"471.542536ms","start":"2026-03-30T21:32:36.376629Z","end":"2026-03-30T21:32:36.848172Z","steps":["trace[1354531051] 'process raft request' (duration: 471.386774ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:32:36.849648Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":16727104867003361064,"received-request-id":16727104867003361062} {"level":"warn","ts":"2026-03-30T21:32:36.849290Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:32:36.376602Z","time spent":"471.656418ms","remote":"127.0.0.1:41456","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4463,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:32:36.853901Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"323.73357ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:32:36.853987Z","caller":"traceutil/trace.go:172","msg":"trace[1815537689] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1708; }","duration":"323.834072ms","start":"2026-03-30T21:32:36.530140Z","end":"2026-03-30T21:32:36.853975Z","steps":["trace[1815537689] 'agreement among raft nodes before linearized reading' (duration: 323.611878ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:32:36.854028Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:32:36.530115Z","time spent":"323.906982ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:32:36.854393Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"389.48317ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:32:36.854425Z","caller":"traceutil/trace.go:172","msg":"trace[1242464487] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1708; }","duration":"389.51452ms","start":"2026-03-30T21:32:36.464904Z","end":"2026-03-30T21:32:36.854418Z","steps":["trace[1242464487] 'agreement among raft nodes before linearized reading' (duration: 389.47318ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:32:36.854641Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"658.775058ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 ","response":"range_response_count:1 size:494"} {"level":"info","ts":"2026-03-30T21:32:36.854667Z","caller":"traceutil/trace.go:172","msg":"trace[689636447] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:1708; }","duration":"658.802578ms","start":"2026-03-30T21:32:36.195858Z","end":"2026-03-30T21:32:36.854661Z","steps":["trace[689636447] 'agreement among raft nodes before linearized reading' (duration: 658.734367ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:32:36.854684Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:32:36.195836Z","time spent":"658.842969ms","remote":"127.0.0.1:41646","response type":"/etcdserverpb.KV/Range","request count":0,"request size":62,"response count":1,"response size":517,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" limit:1 "}