{"level":"warn","ts":"2026-03-26T14:50:41.600323Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"warn","ts":"2026-03-26T14:50:41.600479Z","caller":"etcdmain/config.go:270","msg":"--snapshot-count is deprecated in 3.6 and will be decommissioned in 3.7."} {"level":"info","ts":"2026-03-26T14:50:41.600492Z","caller":"etcdmain/etcd.go:64","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.204.45.242:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--feature-gates=InitialCorruptCheck=true","--initial-advertise-peer-urls=https://199.204.45.242:2380","--initial-cluster=controller-1=https://199.204.45.157:2380,controller-2=https://199.204.45.242:2380","--initial-cluster-state=existing","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.204.45.242:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.204.45.242:2380","--name=controller-2","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--watch-progress-notify-interval=5s"]} {"level":"info","ts":"2026-03-26T14:50:41.600568Z","caller":"etcdmain/etcd.go:124","msg":"Initialize and start etcd server","data-dir":"/var/lib/etcd","dir-type":"empty"} {"level":"warn","ts":"2026-03-26T14:50:41.600587Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-26T14:50:41.600600Z","caller":"embed/etcd.go:138","msg":"configuring peer listeners","listen-peer-urls":["https://199.204.45.242:2380"]} {"level":"info","ts":"2026-03-26T14:50:41.600618Z","caller":"embed/etcd.go:544","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-26T14:50:41.601817Z","caller":"embed/etcd.go:146","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.242:2379"]} {"level":"info","ts":"2026-03-26T14:50:41.601968Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.6.4","git-sha":"5400cdc","go-version":"go1.23.11","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-2","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.204.45.242:2380"],"listen-peer-urls":["https://199.204.45.242:2380"],"advertise-client-urls":["https://199.204.45.242:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.242:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"experimental-local-address":"","cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://199.204.45.157:2380,controller-2=https://199.204.45.242:2380","initial-cluster-state":"existing","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"feature-gates":"InitialCorruptCheck=true","initial-corrupt-check":false,"corrupt-check-time-interval":"0s","compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","discovery-token":"","discovery-endpoints":"","discovery-dial-timeout":"2s","discovery-request-timeout":"5s","discovery-keepalive-time":"2s","discovery-keepalive-timeout":"6s","discovery-insecure-transport":true,"discovery-insecure-skip-tls-verify":false,"discovery-cert":"","discovery-key":"","discovery-cacert":"","discovery-user":"","downgrade-check-interval":"5s","max-learners":1,"v2-deprecation":"write-only"} {"level":"info","ts":"2026-03-26T14:50:41.605611Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc0000daa90}"} {"level":"info","ts":"2026-03-26T14:50:41.609360Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-26T14:50:41.609423Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"4.293482ms"} {"level":"info","ts":"2026-03-26T14:50:41.609455Z","caller":"etcdserver/bootstrap.go:220","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-03-26T14:50:41.609489Z","caller":"etcdserver/bootstrap.go:94","msg":"bootstrapping cluster"} {"level":"info","ts":"2026-03-26T14:50:41.621914Z","caller":"netutil/netutil.go:113","msg":"resolved URL Host","url":"https://199.204.45.242:2380","host":"199.204.45.242:2380","resolved-addr":"199.204.45.242:2380"} {"level":"info","ts":"2026-03-26T14:50:41.621966Z","caller":"netutil/netutil.go:113","msg":"resolved URL Host","url":"https://199.204.45.157:2380","host":"199.204.45.157:2380","resolved-addr":"199.204.45.157:2380"} {"level":"info","ts":"2026-03-26T14:50:41.652190Z","caller":"etcdserver/bootstrap.go:101","msg":"bootstrapping storage"} {"level":"info","ts":"2026-03-26T14:50:41.662243Z","caller":"etcdserver/bootstrap.go:109","msg":"bootstrapping raft"} {"level":"info","ts":"2026-03-26T14:50:41.662279Z","caller":"etcdserver/bootstrap.go:499","msg":"starting local member","local-member-id":"b59fd64328430f5b","cluster-id":"44a3176ec85feb9"} {"level":"info","ts":"2026-03-26T14:50:41.662304Z","caller":"etcdserver/server.go:312","msg":"bootstrap successfully"} {"level":"info","ts":"2026-03-26T14:50:41.662388Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"b59fd64328430f5b switched to configuration voters=()"} {"level":"info","ts":"2026-03-26T14:50:41.662419Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"b59fd64328430f5b became follower at term 0"} {"level":"info","ts":"2026-03-26T14:50:41.662428Z","logger":"raft","caller":"v3@v3.6.0/raft.go:493","msg":"newRaft b59fd64328430f5b [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"warn","ts":"2026-03-26T14:50:41.668681Z","caller":"auth/store.go:1135","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-26T14:50:41.669929Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-26T14:50:41.675358Z","caller":"storage/quota.go:93","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-26T14:50:41.676285Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.676333Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc","remote-peer-urls":["https://199.204.45.157:2380"]} {"level":"info","ts":"2026-03-26T14:50:41.676356Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.676374Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.679036Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.679554Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.679961Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.680009Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc","remote-peer-urls":["https://199.204.45.157:2380"]} {"level":"info","ts":"2026-03-26T14:50:41.680065Z","caller":"etcdserver/server.go:598","msg":"starting etcd server","local-member-id":"b59fd64328430f5b","local-server-version":"3.6.4","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-26T14:50:41.680069Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.680091Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.680244Z","caller":"embed/etcd.go:766","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-26T14:50:41.680306Z","caller":"etcdserver/server.go:499","msg":"starting initial election tick advance","election-ticks":10} {"level":"info","ts":"2026-03-26T14:50:41.680398Z","caller":"embed/etcd.go:640","msg":"serving peer traffic","address":"199.204.45.242:2380"} {"level":"info","ts":"2026-03-26T14:50:41.680453Z","caller":"embed/etcd.go:611","msg":"cmux::serve","address":"199.204.45.242:2380"} {"level":"info","ts":"2026-03-26T14:50:41.680400Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-26T14:50:41.680586Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-26T14:50:41.680595Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"b59fd64328430f5b","initial-advertise-peer-urls":["https://199.204.45.242:2380"],"listen-peer-urls":["https://199.204.45.242:2380"],"advertise-client-urls":["https://199.204.45.242:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.242:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-26T14:50:41.681171Z","caller":"embed/etcd.go:890","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-26T14:50:41.681328Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"b59fd64328430f5b [term: 0] received a MsgHeartbeat message with higher term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:50:41.681401Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"b59fd64328430f5b became follower at term 2"} {"level":"info","ts":"2026-03-26T14:50:41.681433Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: b59fd64328430f5b elected leader 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:50:41.681569Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"b59fd64328430f5b","to":"5f170c75a1d97cc","stream-type":"stream Message"} {"level":"info","ts":"2026-03-26T14:50:41.681607Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.681651Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.680614Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-26T14:50:41.692740Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"b59fd64328430f5b","to":"5f170c75a1d97cc","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-26T14:50:41.692762Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.693299Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:41.693636Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"warn","ts":"2026-03-26T14:50:42.476698Z","caller":"etcdserver/server.go:2304","msg":"Failed to detect schema version","error":"missing confstate information"} {"level":"info","ts":"2026-03-26T14:50:42.787909Z","caller":"rafthttp/http.go:258","msg":"receiving database snapshot","local-member-id":"b59fd64328430f5b","remote-snapshot-sender-id":"5f170c75a1d97cc","incoming-snapshot-index":530,"incoming-snapshot-message-size-bytes":8195,"incoming-snapshot-message-size":"8.2 kB"} {"level":"info","ts":"2026-03-26T14:50:42.801540Z","caller":"snap/db.go:65","msg":"saved database snapshot to disk","path":"/var/lib/etcd/member/snap/0000000000000212.snap.db","bytes":802816,"size":"803 kB"} {"level":"info","ts":"2026-03-26T14:50:42.801604Z","caller":"rafthttp/http.go:287","msg":"received and saved database snapshot","local-member-id":"b59fd64328430f5b","remote-snapshot-sender-id":"5f170c75a1d97cc","incoming-snapshot-index":530,"incoming-snapshot-size-bytes":802816,"incoming-snapshot-size":"803 kB","download-took":"13.747839ms"} {"level":"info","ts":"2026-03-26T14:50:42.801807Z","logger":"raft","caller":"v3@v3.6.0/log.go:465","msg":"log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 530, term: 2]"} {"level":"info","ts":"2026-03-26T14:50:42.801887Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"b59fd64328430f5b switched to configuration voters=(428247441089599436) learners=(13087414626088587099)"} {"level":"info","ts":"2026-03-26T14:50:42.801913Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1935","msg":"b59fd64328430f5b [commit: 530, lastindex: 530, lastterm: 2] restored snapshot [index: 530, term: 2]"} {"level":"info","ts":"2026-03-26T14:50:42.801930Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1844","msg":"b59fd64328430f5b [commit: 530] restored snapshot [index: 530, term: 2]"} {"level":"info","ts":"2026-03-26T14:50:42.802071Z","caller":"etcdserver/server.go:1007","msg":"applying snapshot","current-snapshot-index":0,"current-applied-index":0,"incoming-leader-snapshot-index":530,"incoming-leader-snapshot-term":2} {"level":"info","ts":"2026-03-26T14:50:42.806016Z","caller":"etcdserver/raft.go:276","msg":"applied incoming Raft snapshot","snapshot-index":530} {"level":"info","ts":"2026-03-26T14:50:42.806473Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc0003c6058}"} {"level":"info","ts":"2026-03-26T14:50:42.806989Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-26T14:50:42.807032Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"641.722µs"} {"level":"info","ts":"2026-03-26T14:50:42.807209Z","caller":"etcdserver/server.go:1055","msg":"restoring lease store"} {"level":"info","ts":"2026-03-26T14:50:42.808256Z","caller":"etcdserver/server.go:1059","msg":"restored lease store"} {"level":"info","ts":"2026-03-26T14:50:42.808284Z","caller":"etcdserver/server.go:1062","msg":"restoring mvcc store"} {"level":"info","ts":"2026-03-26T14:50:42.815480Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":504} {"level":"info","ts":"2026-03-26T14:50:42.815546Z","caller":"etcdserver/server.go:1070","msg":"restored mvcc store","consistent-index":530} {"level":"info","ts":"2026-03-26T14:50:42.815564Z","caller":"etcdserver/server.go:1090","msg":"restoring alarm store"} {"level":"info","ts":"2026-03-26T14:50:42.816018Z","caller":"etcdserver/server.go:1078","msg":"closing old backend file"} {"level":"info","ts":"2026-03-26T14:50:42.818000Z","caller":"etcdserver/server.go:1096","msg":"restored alarm store"} {"level":"info","ts":"2026-03-26T14:50:42.818044Z","caller":"etcdserver/server.go:1099","msg":"restoring auth store"} {"level":"info","ts":"2026-03-26T14:50:42.818082Z","caller":"etcdserver/server.go:1103","msg":"restored auth store"} {"level":"info","ts":"2026-03-26T14:50:42.818090Z","caller":"etcdserver/server.go:1106","msg":"restoring v2 store"} {"level":"info","ts":"2026-03-26T14:50:42.818488Z","caller":"etcdserver/server.go:1080","msg":"closed old backend file"} {"level":"info","ts":"2026-03-26T14:50:42.818755Z","caller":"etcdserver/server.go:1115","msg":"restored v2 store"} {"level":"info","ts":"2026-03-26T14:50:42.818802Z","caller":"etcdserver/server.go:1119","msg":"restoring cluster configuration"} {"level":"info","ts":"2026-03-26T14:50:42.818872Z","caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.6"} {"level":"info","ts":"2026-03-26T14:50:42.818898Z","caller":"membership/cluster.go:297","msg":"recovered/added member from store","cluster-id":"44a3176ec85feb9","local-member-id":"b59fd64328430f5b","recovered-remote-peer-id":"5f170c75a1d97cc","recovered-remote-peer-urls":["https://199.204.45.157:2380"],"recovered-remote-peer-is-learner":false} {"level":"info","ts":"2026-03-26T14:50:42.818913Z","caller":"membership/cluster.go:297","msg":"recovered/added member from store","cluster-id":"44a3176ec85feb9","local-member-id":"b59fd64328430f5b","recovered-remote-peer-id":"b59fd64328430f5b","recovered-remote-peer-urls":["https://199.204.45.242:2380"],"recovered-remote-peer-is-learner":true} {"level":"info","ts":"2026-03-26T14:50:42.818925Z","caller":"membership/cluster.go:307","msg":"set cluster version from store","cluster-version":"3.6"} {"level":"info","ts":"2026-03-26T14:50:42.818935Z","caller":"etcdserver/server.go:1123","msg":"restored cluster configuration"} {"level":"info","ts":"2026-03-26T14:50:42.818945Z","caller":"etcdserver/server.go:1124","msg":"removing old peers from network"} {"level":"info","ts":"2026-03-26T14:50:42.818959Z","caller":"rafthttp/peer.go:316","msg":"stopping remote peer","remote-peer-id":"5f170c75a1d97cc"} {"level":"warn","ts":"2026-03-26T14:50:42.818988Z","caller":"rafthttp/stream.go:285","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819015Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"5f170c75a1d97cc"} {"level":"warn","ts":"2026-03-26T14:50:42.819067Z","caller":"rafthttp/stream.go:285","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819084Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819350Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"warn","ts":"2026-03-26T14:50:42.819486Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc","error":"context canceled"} {"level":"warn","ts":"2026-03-26T14:50:42.819524Z","caller":"rafthttp/peer_status.go:66","msg":"peer became inactive (message send to peer failed)","peer-id":"5f170c75a1d97cc","error":"failed to read 5f170c75a1d97cc on stream MsgApp v2 (context canceled)"} {"level":"info","ts":"2026-03-26T14:50:42.819547Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"warn","ts":"2026-03-26T14:50:42.819643Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:50:42.819695Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819721Z","caller":"rafthttp/peer.go:321","msg":"stopped remote peer","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819737Z","caller":"rafthttp/transport.go:354","msg":"removed remote peer","local-member-id":"b59fd64328430f5b","removed-remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819751Z","caller":"etcdserver/server.go:1129","msg":"removed old peers from network"} {"level":"info","ts":"2026-03-26T14:50:42.819759Z","caller":"etcdserver/server.go:1130","msg":"adding peers from new cluster configuration"} {"level":"info","ts":"2026-03-26T14:50:42.819778Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819840Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819894Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.819945Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc","remote-peer-urls":["https://199.204.45.157:2380"]} {"level":"info","ts":"2026-03-26T14:50:42.819966Z","caller":"etcdserver/server.go:1139","msg":"added peers from new cluster configuration"} {"level":"info","ts":"2026-03-26T14:50:42.819985Z","caller":"etcdserver/server.go:1015","msg":"applied snapshot","current-snapshot-index":530,"current-applied-index":530,"incoming-leader-snapshot-index":530,"incoming-leader-snapshot-term":2} {"level":"info","ts":"2026-03-26T14:50:42.820017Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.820035Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.820043Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.820216Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.831793Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"b59fd64328430f5b","to":"5f170c75a1d97cc","stream-type":"stream Message"} {"level":"info","ts":"2026-03-26T14:50:42.831830Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.831856Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.831891Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.832053Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"b59fd64328430f5b","to":"5f170c75a1d97cc","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-26T14:50:42.832073Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:42.836851Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5f170c75a1d97cc"} {"level":"info","ts":"2026-03-26T14:50:43.272918Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"b59fd64328430f5b switched to configuration voters=(428247441089599436 13087414626088587099)"} {"level":"info","ts":"2026-03-26T14:50:43.273114Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"44a3176ec85feb9","local-member-id":"b59fd64328430f5b","promoted-member-id":"b59fd64328430f5b"} {"level":"warn","ts":"2026-03-26T14:50:48.680657Z","caller":"etcdserver/server.go:1814","msg":"failed to publish local member to cluster through raft","local-member-id":"b59fd64328430f5b","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.242:2379]}","publish-timeout":"7s","error":"etcdserver: request timed out, possibly due to connection lost"} {"level":"info","ts":"2026-03-26T14:50:48.721783Z","caller":"etcdserver/server.go:1804","msg":"published local member to cluster through raft","local-member-id":"b59fd64328430f5b","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.242:2379]}","cluster-id":"44a3176ec85feb9","publish-timeout":"7s"} {"level":"info","ts":"2026-03-26T14:50:48.721892Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-26T14:50:48.721994Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-26T14:50:48.722138Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-26T14:50:48.722150Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-03-26T14:50:48.723365Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"warn","ts":"2026-03-26T14:50:48.724077Z","caller":"v3rpc/grpc.go:52","msg":"etcdserver: failed to register grpc metrics","error":"duplicate metrics collector registration attempted"} {"level":"info","ts":"2026-03-26T14:50:48.724313Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-26T14:50:48.726484Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.204.45.242:2379"} {"level":"info","ts":"2026-03-26T14:50:48.728707Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"warn","ts":"2026-03-26T14:50:48.729430Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38924","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.732519Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46196","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.733290Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38940","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.734462Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46182","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.734621Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46166","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.734759Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46170","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.735779Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46152","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.735906Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46146","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.736233Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38910","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.736681Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46200","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.737180Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46132","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.738266Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46180","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.738579Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.242:46184","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.744077Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49206","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.757469Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49236","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.773163Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49248","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.788324Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49276","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.796850Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49286","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.806394Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49290","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.814989Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49302","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.826237Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49320","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.834099Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49336","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.842176Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49346","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.851168Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49358","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.859179Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49374","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.869240Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49398","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.880177Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49408","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.894206Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49424","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.905309Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49436","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.914328Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49464","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.965900Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49468","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.974276Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49492","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.982386Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49504","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.991825Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49536","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:48.999945Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49540","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.009455Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49554","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.017538Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49582","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.039628Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49622","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.047526Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49634","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.057416Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49652","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.065653Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49676","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.074646Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49704","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.081384Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49714","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.092656Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49716","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.100749Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49720","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.109896Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49746","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.121878Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49764","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.158279Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49776","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.176749Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49788","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.186835Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49804","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.196101Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49820","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.204897Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49826","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.213337Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49846","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.221141Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49874","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.231532Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49894","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.238870Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49928","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.247551Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49944","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.255177Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49960","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.263549Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49970","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.270221Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49996","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.278683Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:50018","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.286225Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:50040","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.296418Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:50046","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.310386Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:50070","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.354412Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:50106","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.375803Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:50146","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.424750Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:50162","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:49.511484Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"126.565791ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:49.511596Z","caller":"traceutil/trace.go:172","msg":"trace[1560066700] range","detail":"{range_begin:/registry/resourceslices; range_end:; response_count:0; response_revision:519; }","duration":"126.693163ms","start":"2026-03-26T14:50:49.384891Z","end":"2026-03-26T14:50:49.511584Z","steps":["trace[1560066700] 'range keys from in-memory index tree' (duration: 124.230394ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:49.511838Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"126.759994ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices/\" range_end:\"/registry/resourceslices0\" limit:10000 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:49.513516Z","caller":"traceutil/trace.go:172","msg":"trace[1759281015] range","detail":"{range_begin:/registry/resourceslices/; range_end:/registry/resourceslices0; response_count:0; response_revision:519; }","duration":"128.437801ms","start":"2026-03-26T14:50:49.385069Z","end":"2026-03-26T14:50:49.513507Z","steps":["trace[1759281015] 'range keys from in-memory index tree' (duration: 123.35437ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:51.414276Z","caller":"traceutil/trace.go:172","msg":"trace[160139151] linearizableReadLoop","detail":"{readStateIndex:553; appliedIndex:554; }","duration":"111.683582ms","start":"2026-03-26T14:50:51.302563Z","end":"2026-03-26T14:50:51.414247Z","steps":["trace[160139151] 'read index received' (duration: 111.677662ms)","trace[160139151] 'applied index is now lower than readState.Index' (duration: 4.51µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:51.414419Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"111.842754ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.242\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:51.414605Z","caller":"traceutil/trace.go:172","msg":"trace[804575720] range","detail":"{range_begin:/registry/masterleases/199.204.45.242; range_end:; response_count:0; response_revision:524; }","duration":"112.036487ms","start":"2026-03-26T14:50:51.302554Z","end":"2026-03-26T14:50:51.414591Z","steps":["trace[804575720] 'agreement among raft nodes before linearized reading' (duration: 111.817484ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:59.610741Z","caller":"traceutil/trace.go:172","msg":"trace[166494868] linearizableReadLoop","detail":"{readStateIndex:581; appliedIndex:582; }","duration":"119.471364ms","start":"2026-03-26T14:50:59.491239Z","end":"2026-03-26T14:50:59.610710Z","steps":["trace[166494868] 'read index received' (duration: 119.466163ms)","trace[166494868] 'applied index is now lower than readState.Index' (duration: 4.021µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:59.620120Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"128.858043ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csinodes/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:59.620208Z","caller":"traceutil/trace.go:172","msg":"trace[1749956907] range","detail":"{range_begin:/registry/csinodes/controller-2; range_end:; response_count:0; response_revision:549; }","duration":"128.963645ms","start":"2026-03-26T14:50:59.491226Z","end":"2026-03-26T14:50:59.620189Z","steps":["trace[1749956907] 'agreement among raft nodes before linearized reading' (duration: 119.650066ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:01.817897Z","caller":"traceutil/trace.go:172","msg":"trace[1914984701] transaction","detail":"{read_only:false; response_revision:574; number_of_response:1; }","duration":"117.23588ms","start":"2026-03-26T14:51:01.700645Z","end":"2026-03-26T14:51:01.817881Z","steps":["trace[1914984701] 'process raft request' (duration: 117.112988ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:01.818337Z","caller":"traceutil/trace.go:172","msg":"trace[1375668694] linearizableReadLoop","detail":"{readStateIndex:608; appliedIndex:609; }","duration":"117.894699ms","start":"2026-03-26T14:51:01.700422Z","end":"2026-03-26T14:51:01.818317Z","steps":["trace[1375668694] 'read index received' (duration: 117.881759ms)","trace[1375668694] 'applied index is now lower than readState.Index' (duration: 12.05µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:01.818497Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"118.057182ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/kube-system/kube-proxy\" limit:1 ","response":"range_response_count:1 size:185"} {"level":"info","ts":"2026-03-26T14:51:01.818529Z","caller":"traceutil/trace.go:172","msg":"trace[472426842] range","detail":"{range_begin:/registry/serviceaccounts/kube-system/kube-proxy; range_end:; response_count:1; response_revision:574; }","duration":"118.100843ms","start":"2026-03-26T14:51:01.700419Z","end":"2026-03-26T14:51:01.818520Z","steps":["trace[472426842] 'agreement among raft nodes before linearized reading' (duration: 117.980041ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:10.864239Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:10.963165Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:10.963275Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:10.963314Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:10.963334Z","logger":"raft","caller":"v3@v3.6.0/node.go:376","msg":"raft.node: b59fd64328430f5b lost leader 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:10.963388Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:10.963410Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:11.365494Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:11.866651Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:11.963426Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:11.963494Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:11.963522Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:11.963567Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:11.963584Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:12.364733Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.001000411s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:12.364818Z","caller":"traceutil/trace.go:172","msg":"trace[824990694] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.001100243s","start":"2026-03-26T14:51:10.363699Z","end":"2026-03-26T14:51:12.364799Z","steps":["trace[824990694] 'agreement among raft nodes before linearized reading' (duration: 2.000996771s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:12.364870Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.363674Z","time spent":"2.001187713s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:12 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:12.367014Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:12.868169Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:12.962921Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:12.962982Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:12.963013Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:12.963055Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:12.963071Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:13.368316Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:13.600228Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999920972s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:13.600320Z","caller":"traceutil/trace.go:172","msg":"trace[1843099786] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000026024s","start":"2026-03-26T14:51:11.600272Z","end":"2026-03-26T14:51:13.600298Z","steps":["trace[1843099786] 'agreement among raft nodes before linearized reading' (duration: 1.999917512s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:13.600383Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.600251Z","time spent":"2.000120156s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:13.868577Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:13.963918Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:13.967070Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:13.967120Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:13.967439Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:13.967463Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:14.364863Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999612319s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:14.364933Z","caller":"traceutil/trace.go:172","msg":"trace[739944997] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.999726901s","start":"2026-03-26T14:51:12.365191Z","end":"2026-03-26T14:51:14.364918Z","steps":["trace[739944997] 'agreement among raft nodes before linearized reading' (duration: 1.999609799s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:14.365381Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:12.365170Z","time spent":"2.000198597s","remote":"127.0.0.1:38928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:14 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:14.370742Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:14.872791Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:14.962490Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:14.962541Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:14.962560Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:14.962587Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:14.962597Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:15.373298Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:15.394164Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999152755s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:15.394212Z","caller":"traceutil/trace.go:172","msg":"trace[511837479] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999215196s","start":"2026-03-26T14:51:10.394989Z","end":"2026-03-26T14:51:15.394204Z","steps":["trace[511837479] 'agreement among raft nodes before linearized reading' (duration: 4.999152525s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:15.394239Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.394967Z","time spent":"4.999266956s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:15.873902Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:15.963851Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:15.963907Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:15.963926Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:15.963996Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:15.964012Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:16.306864Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999198691s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:16.306955Z","caller":"traceutil/trace.go:172","msg":"trace[739914526] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"4.999305343s","start":"2026-03-26T14:51:11.307641Z","end":"2026-03-26T14:51:16.306947Z","steps":["trace[739914526] 'agreement among raft nodes before linearized reading' (duration: 4.999197071s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:16.307028Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.307626Z","time spent":"4.999393844s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:16.379050Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.001538995s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:16.379166Z","caller":"traceutil/trace.go:172","msg":"trace[144480730] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.001659877s","start":"2026-03-26T14:51:14.377470Z","end":"2026-03-26T14:51:16.379130Z","steps":["trace[144480730] 'agreement among raft nodes before linearized reading' (duration: 2.001537195s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:16.379221Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:14.377459Z","time spent":"2.001753378s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:16.382119Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:16.882207Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143956,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:16.932953Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:09.932868Z","time spent":"7.00008077s","remote":"127.0.0.1:49420","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"info","ts":"2026-03-26T14:51:16.962661Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:16.962710Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:16.962730Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:16.962787Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:16.962806Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:17.363964Z","caller":"etcdserver/v3_server.go:923","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"} {"level":"warn","ts":"2026-03-26T14:51:17.721677Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.721606Z","time spent":"7.00006564s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:17.867538Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:17.962506Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:17.962553Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:17.962570Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:17.962727Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:17.962744Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:18.367767Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:18.381978Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.995775104s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:18.382345Z","caller":"traceutil/trace.go:172","msg":"trace[1823050626] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.996148279s","start":"2026-03-26T14:51:16.386183Z","end":"2026-03-26T14:51:18.382332Z","steps":["trace[1823050626] 'agreement among raft nodes before linearized reading' (duration: 1.995773514s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:18.382601Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:16.386169Z","time spent":"1.996422412s","remote":"127.0.0.1:38928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:18 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:18.870911Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:18.962712Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:18.962785Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:18.962803Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:18.962828Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:18.962838Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:19.371021Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:19.871438Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:19.962488Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:19.962545Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:19.962564Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:19.962594Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:19.962605Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:20.371523Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:20.386336Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999771093s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:20.386404Z","caller":"traceutil/trace.go:172","msg":"trace[403756822] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000115557s","start":"2026-03-26T14:51:18.386275Z","end":"2026-03-26T14:51:20.386391Z","steps":["trace[403756822] 'agreement among raft nodes before linearized reading' (duration: 1.999770123s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:20.386532Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:18.386264Z","time spent":"2.0002592s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:20.604563Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:13.603446Z","time spent":"7.001112488s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:20.873959Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:20.962484Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:20.962546Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:20.962567Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:20.962597Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:20.962614Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:21.375856Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:21.878531Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:21.964084Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:21.964131Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:21.964149Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:21.964174Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:21.964187Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:22.383092Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:22.388602Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.001951018s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:22.388663Z","caller":"traceutil/trace.go:172","msg":"trace[941579339] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.002010068s","start":"2026-03-26T14:51:20.386632Z","end":"2026-03-26T14:51:22.388642Z","steps":["trace[941579339] 'agreement among raft nodes before linearized reading' (duration: 2.001948888s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:22.388698Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:20.386618Z","time spent":"2.002073999s","remote":"127.0.0.1:38928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:22 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:22.883165Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:22.962720Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:22.962772Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:22.962790Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:22.962815Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:22.962824Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:23.384235Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:23.599520Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999546904s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:23.599584Z","caller":"traceutil/trace.go:172","msg":"trace[1775564869] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.999617215s","start":"2026-03-26T14:51:21.599955Z","end":"2026-03-26T14:51:23.599572Z","steps":["trace[1775564869] 'agreement among raft nodes before linearized reading' (duration: 1.999545994s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.599811Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:21.599660Z","time spent":"2.000143032s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:23.884314Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143960,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:23.908765Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999219955s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:23.908817Z","caller":"traceutil/trace.go:172","msg":"trace[1623274824] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999289256s","start":"2026-03-26T14:51:18.909519Z","end":"2026-03-26T14:51:23.908808Z","steps":["trace[1623274824] 'agreement among raft nodes before linearized reading' (duration: 4.999218745s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.910572Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:18.909507Z","time spent":"5.001055068s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:23.962505Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:23.962553Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:23.962571Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:23.962604Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:23.962631Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:24.364225Z","caller":"etcdserver/v3_server.go:923","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"} {"level":"warn","ts":"2026-03-26T14:51:24.364355Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.424659905s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions/controller-2\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:24.364380Z","caller":"traceutil/trace.go:172","msg":"trace[690375717] range","detail":"{range_begin:/registry/minions/controller-2; range_end:; }","duration":"7.424701465s","start":"2026-03-26T14:51:16.939668Z","end":"2026-03-26T14:51:24.364369Z","steps":["trace[690375717] 'agreement among raft nodes before linearized reading' (duration: 7.424658995s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:24.364412Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:16.939654Z","time spent":"7.424747566s","remote":"127.0.0.1:49420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":0,"request content":"key:\"/registry/minions/controller-2\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:24.364439Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"13.737113488s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:24.364452Z","caller":"traceutil/trace.go:172","msg":"trace[720557896] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; }","duration":"13.737127508s","start":"2026-03-26T14:51:10.627320Z","end":"2026-03-26T14:51:24.364448Z","steps":["trace[720557896] 'agreement among raft nodes before linearized reading' (duration: 13.737113668s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:24.364436Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"13.061194967s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.242\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:24.364604Z","caller":"traceutil/trace.go:172","msg":"trace[866603564] range","detail":"{range_begin:/registry/masterleases/199.204.45.242; range_end:; }","duration":"13.061378019s","start":"2026-03-26T14:51:11.303213Z","end":"2026-03-26T14:51:24.364591Z","steps":["trace[866603564] 'agreement among raft nodes before linearized reading' (duration: 13.061193487s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:24.364639Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.303189Z","time spent":"13.06144095s","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":0,"response size":0,"request content":"key:\"/registry/masterleases/199.204.45.242\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:24.364673Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.642937294s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-proxy-xts8v\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:24.364713Z","caller":"traceutil/trace.go:172","msg":"trace[1777074780] range","detail":"{range_begin:/registry/pods/kube-system/kube-proxy-xts8v; range_end:; }","duration":"7.642974805s","start":"2026-03-26T14:51:16.721727Z","end":"2026-03-26T14:51:24.364702Z","steps":["trace[1777074780] 'agreement among raft nodes before linearized reading' (duration: 7.642936954s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:24.364989Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:16.721716Z","time spent":"7.643259369s","remote":"127.0.0.1:49434","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/pods/kube-system/kube-proxy-xts8v\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:24.364688Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"12.390364223s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:24.365034Z","caller":"traceutil/trace.go:172","msg":"trace[212439192] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; }","duration":"12.390711279s","start":"2026-03-26T14:51:11.974317Z","end":"2026-03-26T14:51:24.365028Z","steps":["trace[212439192] 'agreement among raft nodes before linearized reading' (duration: 12.390364233s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:24.365056Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.974293Z","time spent":"12.390756909s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:24.364544Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"13.816373011s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:24.365220Z","caller":"traceutil/trace.go:172","msg":"trace[140204682] range","detail":"{range_begin:; range_end:; }","duration":"13.817052261s","start":"2026-03-26T14:51:10.548147Z","end":"2026-03-26T14:51:24.365199Z","steps":["trace[140204682] 'agreement among raft nodes before linearized reading' (duration: 13.816370791s)"],"step_count":1} {"level":"error","ts":"2026-03-26T14:51:24.365331Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: request timed out\n[+]non_learner ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-26T14:51:24.364469Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.627289Z","time spent":"13.73717596s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:24.392748Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000393668s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:24.393056Z","caller":"traceutil/trace.go:172","msg":"trace[2037910379] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000451268s","start":"2026-03-26T14:51:22.392336Z","end":"2026-03-26T14:51:24.392788Z","steps":["trace[2037910379] 'agreement among raft nodes before linearized reading' (duration: 2.000391848s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:24.393096Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:22.392326Z","time spent":"2.000762233s","remote":"127.0.0.1:38928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:24.733809Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:17.733528Z","time spent":"7.000275902s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:24.805362Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999480039s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:24.805509Z","caller":"traceutil/trace.go:172","msg":"trace[418771989] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"4.99962444s","start":"2026-03-26T14:51:19.805860Z","end":"2026-03-26T14:51:24.805485Z","steps":["trace[418771989] 'agreement among raft nodes before linearized reading' (duration: 4.999478218s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:24.805584Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:19.805846Z","time spent":"4.999730502s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:24.864302Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143964,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:24.962494Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:24.962547Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:24.962570Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:24.963808Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:24.963862Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:25.364752Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143964,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:25.865857Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143964,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:25.962485Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"b59fd64328430f5b is starting a new election at term 2"} {"level":"info","ts":"2026-03-26T14:51:25.962537Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"b59fd64328430f5b became pre-candidate at term 2"} {"level":"info","ts":"2026-03-26T14:51:25.962563Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"b59fd64328430f5b [logterm: 2, index: 641] sent MsgPreVote request to 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:51:25.962599Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"b59fd64328430f5b received MsgPreVoteResp from b59fd64328430f5b at term 2"} {"level":"info","ts":"2026-03-26T14:51:25.962622Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"b59fd64328430f5b has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-26T14:51:26.366930Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143964,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:26.399895Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.997373328s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:26.400887Z","caller":"traceutil/trace.go:172","msg":"trace[1946903955] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.999702169s","start":"2026-03-26T14:51:24.401173Z","end":"2026-03-26T14:51:26.400875Z","steps":["trace[1946903955] 'agreement among raft nodes before linearized reading' (duration: 1.997371578s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:26.400927Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:24.401167Z","time spent":"1.999753249s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:26 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:51:26.660366Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"b59fd64328430f5b became follower at term 2"} {"level":"info","ts":"2026-03-26T14:51:26.660412Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: b59fd64328430f5b elected leader 5f170c75a1d97cc at term 2"} {"level":"warn","ts":"2026-03-26T14:51:26.775372Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:19.774701Z","time spent":"7.000663361s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:26.936468Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.546161781s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions/controller-2\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:26.936555Z","caller":"traceutil/trace.go:172","msg":"trace[442698849] range","detail":"{range_begin:/registry/minions/controller-2; range_end:; }","duration":"2.546279942s","start":"2026-03-26T14:51:24.390259Z","end":"2026-03-26T14:51:26.936539Z","steps":["trace[442698849] 'agreement among raft nodes before linearized reading' (duration: 2.546157461s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:26.936596Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:24.390246Z","time spent":"2.546343724s","remote":"127.0.0.1:49420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":0,"request content":"key:\"/registry/minions/controller-2\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.102494Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853143966,"received-request-id":1106650939853143956} {"level":"warn","ts":"2026-03-26T14:51:27.162000Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143966,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:27.634104Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:20.633164Z","time spent":"7.000933781s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:27.662216Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143966,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:28.163369Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143966,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:28.212478Z","caller":"traceutil/trace.go:172","msg":"trace[1338243242] linearizableReadLoop","detail":"{readStateIndex:641; appliedIndex:655; }","duration":"1.550910933s","start":"2026-03-26T14:51:26.661534Z","end":"2026-03-26T14:51:28.212445Z","steps":["trace[1338243242] 'read index received' (duration: 1.550900673s)","trace[1338243242] 'applied index is now lower than readState.Index' (duration: 8.3µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:28.212634Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.846218881s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:28.212676Z","caller":"traceutil/trace.go:172","msg":"trace[1444681527] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:611; }","duration":"3.846274532s","start":"2026-03-26T14:51:24.366386Z","end":"2026-03-26T14:51:28.212661Z","steps":["trace[1444681527] 'agreement among raft nodes before linearized reading' (duration: 3.846190871s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:28.212856Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.812974713s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:28.212859Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.818231204s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.242\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-26T14:51:28.212902Z","caller":"traceutil/trace.go:172","msg":"trace[1614695343] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:613; }","duration":"1.813021394s","start":"2026-03-26T14:51:26.399868Z","end":"2026-03-26T14:51:28.212890Z","steps":["trace[1614695343] 'agreement among raft nodes before linearized reading' (duration: 1.812940683s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:28.212913Z","caller":"traceutil/trace.go:172","msg":"trace[1238383310] transaction","detail":"{read_only:false; number_of_response:1; response_revision:613; }","duration":"1.433453889s","start":"2026-03-26T14:51:26.779443Z","end":"2026-03-26T14:51:28.212896Z","steps":["trace[1238383310] 'process raft request' (duration: 1.433369658s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:28.212937Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:26.399834Z","time spent":"1.813094414s","remote":"127.0.0.1:38928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:28.213045Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.813789135s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:425"} {"level":"info","ts":"2026-03-26T14:51:28.213101Z","caller":"traceutil/trace.go:172","msg":"trace[971644346] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:614; }","duration":"3.813843336s","start":"2026-03-26T14:51:24.399246Z","end":"2026-03-26T14:51:28.213089Z","steps":["trace[971644346] 'agreement among raft nodes before linearized reading' (duration: 3.813703994s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:28.213105Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.821529147s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:429"} {"level":"info","ts":"2026-03-26T14:51:28.213115Z","caller":"traceutil/trace.go:172","msg":"trace[139233744] transaction","detail":"{read_only:false; number_of_response:1; response_revision:613; }","duration":"3.444380568s","start":"2026-03-26T14:51:24.768724Z","end":"2026-03-26T14:51:28.213104Z","steps":["trace[139233744] 'process raft request' (duration: 3.444043514s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:28.213155Z","caller":"traceutil/trace.go:172","msg":"trace[872820749] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:614; }","duration":"3.821584298s","start":"2026-03-26T14:51:24.391558Z","end":"2026-03-26T14:51:28.213142Z","steps":["trace[872820749] 'agreement among raft nodes before linearized reading' (duration: 3.821390035s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:28.213738Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:24.391551Z","time spent":"3.822180035s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":452,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:28.213140Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:24.399232Z","time spent":"3.813898527s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":448,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:28.213261Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.822026733s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-proxy-xts8v\" limit:1 ","response":"range_response_count:1 size:5123"} {"level":"info","ts":"2026-03-26T14:51:28.213920Z","caller":"traceutil/trace.go:172","msg":"trace[862049700] range","detail":"{range_begin:/registry/pods/kube-system/kube-proxy-xts8v; range_end:; response_count:1; response_revision:614; }","duration":"3.822677021s","start":"2026-03-26T14:51:24.391223Z","end":"2026-03-26T14:51:28.213900Z","steps":["trace[862049700] 'agreement among raft nodes before linearized reading' (duration: 3.821955672s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:28.212926Z","caller":"traceutil/trace.go:172","msg":"trace[1559725679] range","detail":"{range_begin:/registry/masterleases/199.204.45.242; range_end:; response_count:1; response_revision:611; }","duration":"3.818373386s","start":"2026-03-26T14:51:24.394535Z","end":"2026-03-26T14:51:28.212909Z","steps":["trace[1559725679] 'agreement among raft nodes before linearized reading' (duration: 3.818086602s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:28.214163Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:24.391212Z","time spent":"3.822922576s","remote":"127.0.0.1:49434","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":5146,"request content":"key:\"/registry/pods/kube-system/kube-proxy-xts8v\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:28.214501Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:26.779418Z","time spent":"1.433542149s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":49,"response count":0,"response size":586,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:28.214666Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:24.767789Z","time spent":"3.445922699s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":69,"response count":0,"response size":726,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:28.214258Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:24.394521Z","time spent":"3.819717963s","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.242\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:28.712892Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143969,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:29.213515Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143969,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:29.369068Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853143969,"received-request-id":1106650939853143966} {"level":"info","ts":"2026-03-26T14:51:29.372199Z","caller":"traceutil/trace.go:172","msg":"trace[1348635541] transaction","detail":"{read_only:false; response_revision:619; number_of_response:1; }","duration":"1.733981681s","start":"2026-03-26T14:51:27.638198Z","end":"2026-03-26T14:51:29.372179Z","steps":["trace[1348635541] 'process raft request' (duration: 1.733560876s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:29.372319Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:27.638162Z","time spent":"1.734108263s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-26T14:51:29.714111Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143969,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:30.214809Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143969,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:30.221616Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000326988s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:30.221735Z","caller":"traceutil/trace.go:172","msg":"trace[1263039347] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.00047973s","start":"2026-03-26T14:51:28.221239Z","end":"2026-03-26T14:51:30.221718Z","steps":["trace[1263039347] 'agreement among raft nodes before linearized reading' (duration: 2.000322918s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.221804Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.221232Z","time spent":"2.000564501s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:30 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:51:30.229066Z","caller":"traceutil/trace.go:172","msg":"trace[1742704151] linearizableReadLoop","detail":"{readStateIndex:660; appliedIndex:667; }","duration":"2.016488916s","start":"2026-03-26T14:51:28.212562Z","end":"2026-03-26T14:51:30.229051Z","steps":["trace[1742704151] 'read index received' (duration: 2.016485666s)","trace[1742704151] 'applied index is now lower than readState.Index' (duration: 2.48µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:30.229213Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.291579028s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions/controller-2\" limit:1 ","response":"range_response_count:1 size:4042"} {"level":"info","ts":"2026-03-26T14:51:30.229236Z","caller":"traceutil/trace.go:172","msg":"trace[1444615960] range","detail":"{range_begin:/registry/minions/controller-2; range_end:; response_count:1; response_revision:619; }","duration":"3.291617468s","start":"2026-03-26T14:51:26.937611Z","end":"2026-03-26T14:51:30.229228Z","steps":["trace[1444615960] 'agreement among raft nodes before linearized reading' (duration: 3.291497767s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.229253Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:26.937590Z","time spent":"3.291658409s","remote":"127.0.0.1:49420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":1,"response size":4065,"request content":"key:\"/registry/minions/controller-2\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:30.229524Z","caller":"traceutil/trace.go:172","msg":"trace[1428785412] transaction","detail":"{read_only:false; response_revision:621; number_of_response:1; }","duration":"2.008803717s","start":"2026-03-26T14:51:28.220709Z","end":"2026-03-26T14:51:30.229512Z","steps":["trace[1428785412] 'process raft request' (duration: 2.008752636s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.229580Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.215890Z","time spent":"2.01368871s","remote":"127.0.0.1:49216","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:30.229609Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.220690Z","time spent":"2.008869637s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:30.229708Z","caller":"traceutil/trace.go:172","msg":"trace[1005311466] transaction","detail":"{read_only:false; response_revision:622; number_of_response:1; }","duration":"2.007262046s","start":"2026-03-26T14:51:28.222438Z","end":"2026-03-26T14:51:30.229700Z","steps":["trace[1005311466] 'process raft request' (duration: 2.007050683s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:30.229759Z","caller":"traceutil/trace.go:172","msg":"trace[661570000] transaction","detail":"{read_only:false; response_revision:620; number_of_response:1; }","duration":"2.009466585s","start":"2026-03-26T14:51:28.220285Z","end":"2026-03-26T14:51:30.229752Z","steps":["trace[661570000] 'process raft request' (duration: 2.00905388s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.229775Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.222427Z","time spent":"2.007296976s","remote":"127.0.0.1:49434","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5033,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:30.229795Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.220273Z","time spent":"2.009502635s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":427,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:30.229993Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853143975,"received-request-id":1106650939853143969} {"level":"warn","ts":"2026-03-26T14:51:30.230926Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.017088853s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:30.230955Z","caller":"traceutil/trace.go:172","msg":"trace[904489728] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:622; }","duration":"2.017119155s","start":"2026-03-26T14:51:28.213830Z","end":"2026-03-26T14:51:30.230949Z","steps":["trace[904489728] 'agreement among raft nodes before linearized reading' (duration: 2.017076043s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.231390Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.986002773s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-26T14:51:30.231446Z","caller":"traceutil/trace.go:172","msg":"trace[1092967952] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:622; }","duration":"1.986074163s","start":"2026-03-26T14:51:28.245356Z","end":"2026-03-26T14:51:30.231430Z","steps":["trace[1092967952] 'agreement among raft nodes before linearized reading' (duration: 1.9857884s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.231492Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.245341Z","time spent":"1.986142744s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:30.231551Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.010806532s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-node-lease/controller-2\" limit:1 ","response":"range_response_count:1 size:552"} {"level":"info","ts":"2026-03-26T14:51:30.231624Z","caller":"traceutil/trace.go:172","msg":"trace[1646964788] range","detail":"{range_begin:/registry/leases/kube-node-lease/controller-2; range_end:; response_count:1; response_revision:622; }","duration":"2.010882483s","start":"2026-03-26T14:51:28.220733Z","end":"2026-03-26T14:51:30.231616Z","steps":["trace[1646964788] 'agreement among raft nodes before linearized reading' (duration: 2.010708901s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.231648Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.220724Z","time spent":"2.010916903s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":49,"response count":1,"response size":575,"request content":"key:\"/registry/leases/kube-node-lease/controller-2\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:30.232157Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"856.08662ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-proxy-xts8v.18a06bb1410199cb\" limit:1 ","response":"range_response_count:1 size:777"} {"level":"info","ts":"2026-03-26T14:51:30.232478Z","caller":"traceutil/trace.go:172","msg":"trace[1711938277] range","detail":"{range_begin:/registry/events/kube-system/kube-proxy-xts8v.18a06bb1410199cb; range_end:; response_count:1; response_revision:622; }","duration":"856.665667ms","start":"2026-03-26T14:51:29.375753Z","end":"2026-03-26T14:51:30.232419Z","steps":["trace[1711938277] 'agreement among raft nodes before linearized reading' (duration: 855.152467ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.232804Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:29.375736Z","time spent":"857.055091ms","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Range","request count":0,"request size":66,"response count":1,"response size":800,"request content":"key:\"/registry/events/kube-system/kube-proxy-xts8v.18a06bb1410199cb\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:30.233087Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.379506132s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-26T14:51:30.233147Z","caller":"traceutil/trace.go:172","msg":"trace[1203797080] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:622; }","duration":"1.38091405s","start":"2026-03-26T14:51:28.852216Z","end":"2026-03-26T14:51:30.233130Z","steps":["trace[1203797080] 'agreement among raft nodes before linearized reading' (duration: 1.379449081s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.233182Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.852191Z","time spent":"1.38097873s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:30.232341Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.012104429s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/apiserver-zjy5m4qienmmbtxvgklj3tgz6i\" limit:1 ","response":"range_response_count:1 size:692"} {"level":"info","ts":"2026-03-26T14:51:30.233309Z","caller":"traceutil/trace.go:172","msg":"trace[1108026859] range","detail":"{range_begin:/registry/leases/kube-system/apiserver-zjy5m4qienmmbtxvgklj3tgz6i; range_end:; response_count:1; response_revision:622; }","duration":"2.013072441s","start":"2026-03-26T14:51:28.220228Z","end":"2026-03-26T14:51:30.233300Z","steps":["trace[1108026859] 'agreement among raft nodes before linearized reading' (duration: 2.012020088s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.233346Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:28.220209Z","time spent":"2.013127791s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":69,"response count":1,"response size":715,"request content":"key:\"/registry/leases/kube-system/apiserver-zjy5m4qienmmbtxvgklj3tgz6i\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:30.734503Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143976,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:31.039707Z","caller":"traceutil/trace.go:172","msg":"trace[1170297237] transaction","detail":"{read_only:false; number_of_response:1; response_revision:622; }","duration":"808.613229ms","start":"2026-03-26T14:51:30.231079Z","end":"2026-03-26T14:51:31.039693Z","steps":["trace[1170297237] 'process raft request' (duration: 808.565178ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:31.039823Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.231062Z","time spent":"808.70319ms","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":41,"response count":0,"response size":37,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:31.235581Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143976,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:31.736569Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143976,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:31.972941Z","caller":"traceutil/trace.go:172","msg":"trace[397616563] linearizableReadLoop","detail":"{readStateIndex:671; appliedIndex:672; }","duration":"1.739420389s","start":"2026-03-26T14:51:30.233498Z","end":"2026-03-26T14:51:31.972918Z","steps":["trace[397616563] 'read index received' (duration: 1.739413299s)","trace[397616563] 'applied index is now lower than readState.Index' (duration: 5.87µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:31.973270Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.739754123s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-26T14:51:31.973307Z","caller":"traceutil/trace.go:172","msg":"trace[1034433426] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:623; }","duration":"1.739803663s","start":"2026-03-26T14:51:30.233494Z","end":"2026-03-26T14:51:31.973297Z","steps":["trace[1034433426] 'agreement among raft nodes before linearized reading' (duration: 1.739642861s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:31.973335Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.233480Z","time spent":"1.739847614s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:31.973452Z","caller":"txn/util.go:46","msg":"failed to apply request","took":"112.232µs","request":"header: lease_revoke:","response":"size:28","error":"lease not found"} {"level":"info","ts":"2026-03-26T14:51:31.973545Z","caller":"traceutil/trace.go:172","msg":"trace[145526698] transaction","detail":"{read_only:false; response_revision:625; number_of_response:1; }","duration":"1.736143607s","start":"2026-03-26T14:51:30.237386Z","end":"2026-03-26T14:51:31.973529Z","steps":["trace[145526698] 'process raft request' (duration: 1.735786182s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:31.973653Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.237380Z","time spent":"1.736208828s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":676,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:31.973877Z","caller":"traceutil/trace.go:172","msg":"trace[824392018] transaction","detail":"{read_only:false; response_revision:624; number_of_response:1; }","duration":"1.737007777s","start":"2026-03-26T14:51:30.236858Z","end":"2026-03-26T14:51:31.973866Z","steps":["trace[824392018] 'process raft request' (duration: 1.736271587s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:31.973940Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.236846Z","time spent":"1.737064408s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":762,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:31.973694Z","caller":"traceutil/trace.go:172","msg":"trace[1979117959] transaction","detail":"{read_only:false; response_revision:623; number_of_response:1; }","duration":"1.737440714s","start":"2026-03-26T14:51:30.236241Z","end":"2026-03-26T14:51:31.973682Z","steps":["trace[1979117959] 'process raft request' (duration: 1.736731984s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:31.974347Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.236229Z","time spent":"1.73792581s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:32.066113Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853143983,"received-request-id":1106650939853143976} {"level":"warn","ts":"2026-03-26T14:51:32.066237Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:31.040853Z","time spent":"1.025381427s","remote":"127.0.0.1:49216","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-26T14:51:32.234588Z","caller":"traceutil/trace.go:172","msg":"trace[1230891355] linearizableReadLoop","detail":"{readStateIndex:680; appliedIndex:681; }","duration":"261.450068ms","start":"2026-03-26T14:51:31.973113Z","end":"2026-03-26T14:51:32.234563Z","steps":["trace[1230891355] 'read index received' (duration: 261.441548ms)","trace[1230891355] 'applied index is now lower than readState.Index' (duration: 6.77µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:32.234714Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999940454s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:32.234742Z","caller":"traceutil/trace.go:172","msg":"trace[21758644] range","detail":"{range_begin:/registry/services/specs; range_end:; response_count:0; response_revision:628; }","duration":"1.999985164s","start":"2026-03-26T14:51:30.234749Z","end":"2026-03-26T14:51:32.234734Z","steps":["trace[21758644] 'agreement among raft nodes before linearized reading' (duration: 1.999902673s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:32.234772Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.234741Z","time spent":"2.000023445s","remote":"127.0.0.1:49450","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/services/specs\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:32.234763Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.99879464s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:32.234846Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"636.422258ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:32.234904Z","caller":"traceutil/trace.go:172","msg":"trace[1239591967] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:628; }","duration":"636.467598ms","start":"2026-03-26T14:51:31.598415Z","end":"2026-03-26T14:51:32.234883Z","steps":["trace[1239591967] 'agreement among raft nodes before linearized reading' (duration: 636.398807ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:32.234950Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:31.598393Z","time spent":"636.54721ms","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:32.235025Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999285296s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 ","response":"range_response_count:1 size:7816"} {"level":"info","ts":"2026-03-26T14:51:32.235065Z","caller":"traceutil/trace.go:172","msg":"trace[367058600] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-2; range_end:; response_count:1; response_revision:629; }","duration":"1.999324856s","start":"2026-03-26T14:51:30.235731Z","end":"2026-03-26T14:51:32.235056Z","steps":["trace[367058600] 'agreement among raft nodes before linearized reading' (duration: 1.999140264s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:32.234846Z","caller":"traceutil/trace.go:172","msg":"trace[39305988] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:628; }","duration":"1.998896281s","start":"2026-03-26T14:51:30.235931Z","end":"2026-03-26T14:51:32.234827Z","steps":["trace[39305988] 'agreement among raft nodes before linearized reading' (duration: 1.998742899s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:32.235101Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.235715Z","time spent":"1.999375977s","remote":"127.0.0.1:49434","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":1,"response size":7839,"request content":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:32.234753Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.997358941s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-26T14:51:32.235337Z","caller":"traceutil/trace.go:172","msg":"trace[913477617] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:628; }","duration":"1.997963698s","start":"2026-03-26T14:51:30.237362Z","end":"2026-03-26T14:51:32.235326Z","steps":["trace[913477617] 'agreement among raft nodes before linearized reading' (duration: 1.997227489s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:32.235586Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.237355Z","time spent":"1.998156221s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:32.234928Z","caller":"traceutil/trace.go:172","msg":"trace[1016038588] transaction","detail":"{read_only:false; response_revision:629; number_of_response:1; }","duration":"257.312195ms","start":"2026-03-26T14:51:31.977603Z","end":"2026-03-26T14:51:32.234915Z","steps":["trace[1016038588] 'process raft request' (duration: 257.176264ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:32.235417Z","caller":"traceutil/trace.go:172","msg":"trace[516940407] transaction","detail":"{read_only:false; response_revision:630; number_of_response:1; }","duration":"251.953886ms","start":"2026-03-26T14:51:31.983452Z","end":"2026-03-26T14:51:32.235406Z","steps":["trace[516940407] 'process raft request' (duration: 251.42694ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:32.235450Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.994887939s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:32.236483Z","caller":"traceutil/trace.go:172","msg":"trace[72384258] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:629; }","duration":"1.995892911s","start":"2026-03-26T14:51:30.240554Z","end":"2026-03-26T14:51:32.236447Z","steps":["trace[72384258] 'agreement among raft nodes before linearized reading' (duration: 1.994528355s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:32.236545Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.240538Z","time spent":"1.995991753s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:51:32.531469Z","caller":"traceutil/trace.go:172","msg":"trace[1444313849] transaction","detail":"{read_only:false; response_revision:631; number_of_response:1; }","duration":"464.23685ms","start":"2026-03-26T14:51:32.067215Z","end":"2026-03-26T14:51:32.531452Z","steps":["trace[1444313849] 'process raft request' (duration: 464.128749ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:32.531605Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:32.067194Z","time spent":"464.343062ms","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:32.738656Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143987,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:33.239568Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143987,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:33.740690Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143987,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:33.827154Z","caller":"traceutil/trace.go:172","msg":"trace[1437029475] linearizableReadLoop","detail":"{readStateIndex:684; appliedIndex:685; }","duration":"1.589411734s","start":"2026-03-26T14:51:32.237724Z","end":"2026-03-26T14:51:33.827135Z","steps":["trace[1437029475] 'read index received' (duration: 1.589405734s)","trace[1437029475] 'applied index is now lower than readState.Index' (duration: 4.57µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:33.827312Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.589565716s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:33.827375Z","caller":"traceutil/trace.go:172","msg":"trace[120928908] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:631; }","duration":"1.589642647s","start":"2026-03-26T14:51:32.237720Z","end":"2026-03-26T14:51:33.827363Z","steps":["trace[120928908] 'agreement among raft nodes before linearized reading' (duration: 1.589526076s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:33.827417Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:32.237709Z","time spent":"1.589700548s","remote":"127.0.0.1:38928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:51:33.827526Z","caller":"traceutil/trace.go:172","msg":"trace[1306823625] transaction","detail":"{read_only:false; response_revision:632; number_of_response:1; }","duration":"1.587058365s","start":"2026-03-26T14:51:32.240449Z","end":"2026-03-26T14:51:33.827508Z","steps":["trace[1306823625] 'process raft request' (duration: 1.586850122s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:33.827551Z","caller":"traceutil/trace.go:172","msg":"trace[780474781] transaction","detail":"{read_only:false; response_revision:634; number_of_response:1; }","duration":"1.584091507s","start":"2026-03-26T14:51:32.243441Z","end":"2026-03-26T14:51:33.827532Z","steps":["trace[780474781] 'process raft request' (duration: 1.584039467s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:33.828094Z","caller":"traceutil/trace.go:172","msg":"trace[879938996] transaction","detail":"{read_only:false; response_revision:633; number_of_response:1; }","duration":"1.585530685s","start":"2026-03-26T14:51:32.242355Z","end":"2026-03-26T14:51:33.827886Z","steps":["trace[879938996] 'process raft request' (duration: 1.585069549s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:33.828236Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:32.242349Z","time spent":"1.585810449s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":422,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:33.828771Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:32.243432Z","time spent":"1.584164818s","remote":"127.0.0.1:49434","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7993,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:33.828840Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:32.240441Z","time spent":"1.587139766s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":712,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-26T14:51:33.941710Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853143991,"received-request-id":1106650939853143987} {"level":"info","ts":"2026-03-26T14:51:34.011521Z","caller":"traceutil/trace.go:172","msg":"trace[865433901] linearizableReadLoop","detail":"{readStateIndex:688; appliedIndex:690; }","duration":"184.290781ms","start":"2026-03-26T14:51:33.827205Z","end":"2026-03-26T14:51:34.011495Z","steps":["trace[865433901] 'read index received' (duration: 184.283901ms)","trace[865433901] 'applied index is now lower than readState.Index' (duration: 5.5µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:34.011661Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"401.626417ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-26T14:51:34.011722Z","caller":"traceutil/trace.go:172","msg":"trace[2009061501] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:636; }","duration":"401.705308ms","start":"2026-03-26T14:51:33.610005Z","end":"2026-03-26T14:51:34.011710Z","steps":["trace[2009061501] 'agreement among raft nodes before linearized reading' (duration: 401.509575ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.011733Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"654.644243ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-26T14:51:34.011798Z","caller":"traceutil/trace.go:172","msg":"trace[443221065] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:636; }","duration":"654.725384ms","start":"2026-03-26T14:51:33.357056Z","end":"2026-03-26T14:51:34.011782Z","steps":["trace[443221065] 'agreement among raft nodes before linearized reading' (duration: 654.519592ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.011790Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.478176238s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-26T14:51:34.011964Z","caller":"traceutil/trace.go:172","msg":"trace[614384694] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:636; }","duration":"1.47836801s","start":"2026-03-26T14:51:32.533584Z","end":"2026-03-26T14:51:34.011952Z","steps":["trace[614384694] 'agreement among raft nodes before linearized reading' (duration: 1.478038546s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.012011Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:32.533571Z","time spent":"1.478428991s","remote":"127.0.0.1:49406","response type":"/etcdserverpb.KV/Range","request count":0,"request size":51,"response count":1,"response size":465,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:34.011843Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:33.357026Z","time spent":"654.807945ms","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:34.011754Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:33.609979Z","time spent":"401.769679ms","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:34.014235Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.776278647s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:34.014284Z","caller":"traceutil/trace.go:172","msg":"trace[1780633211] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:636; }","duration":"1.776327058s","start":"2026-03-26T14:51:32.237943Z","end":"2026-03-26T14:51:34.014270Z","steps":["trace[1780633211] 'agreement among raft nodes before linearized reading' (duration: 1.773981938s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.014310Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"173.372233ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:34.014347Z","caller":"traceutil/trace.go:172","msg":"trace[28749286] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:637; }","duration":"173.408814ms","start":"2026-03-26T14:51:33.840930Z","end":"2026-03-26T14:51:34.014338Z","steps":["trace[28749286] 'agreement among raft nodes before linearized reading' (duration: 173.356813ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.014536Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"180.809156ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a06bb41491d8fb\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-26T14:51:34.014572Z","caller":"traceutil/trace.go:172","msg":"trace[1377260576] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18a06bb41491d8fb; range_end:; response_count:1; response_revision:637; }","duration":"180.845196ms","start":"2026-03-26T14:51:33.833717Z","end":"2026-03-26T14:51:34.014562Z","steps":["trace[1377260576] 'agreement among raft nodes before linearized reading' (duration: 180.747915ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.014726Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"177.91951ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 ","response":"range_response_count:1 size:8008"} {"level":"info","ts":"2026-03-26T14:51:34.015053Z","caller":"traceutil/trace.go:172","msg":"trace[1878033575] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-2; range_end:; response_count:1; response_revision:637; }","duration":"178.248464ms","start":"2026-03-26T14:51:33.836796Z","end":"2026-03-26T14:51:34.015045Z","steps":["trace[1878033575] 'agreement among raft nodes before linearized reading' (duration: 177.85437ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:34.505101Z","caller":"traceutil/trace.go:172","msg":"trace[552284697] transaction","detail":"{read_only:false; response_revision:638; number_of_response:1; }","duration":"489.121474ms","start":"2026-03-26T14:51:34.015962Z","end":"2026-03-26T14:51:34.505083Z","steps":["trace[552284697] 'process raft request' (duration: 488.981702ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.505240Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.015941Z","time spent":"489.230985ms","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:34.505581Z","caller":"traceutil/trace.go:172","msg":"trace[982879391] linearizableReadLoop","detail":"{readStateIndex:691; appliedIndex:692; }","duration":"489.64614ms","start":"2026-03-26T14:51:34.015926Z","end":"2026-03-26T14:51:34.505572Z","steps":["trace[982879391] 'read index received' (duration: 489.64227ms)","trace[982879391] 'applied index is now lower than readState.Index' (duration: 3.01µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:34.505650Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"489.978304ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:34.505668Z","caller":"traceutil/trace.go:172","msg":"trace[227563396] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:638; }","duration":"489.998704ms","start":"2026-03-26T14:51:34.015664Z","end":"2026-03-26T14:51:34.505662Z","steps":["trace[227563396] 'agreement among raft nodes before linearized reading' (duration: 489.962074ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.507424Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"491.370292ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:34.507664Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"486.48344ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 ","response":"range_response_count:1 size:8008"} {"level":"warn","ts":"2026-03-26T14:51:34.507666Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"486.341589ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:498"} {"level":"info","ts":"2026-03-26T14:51:34.507707Z","caller":"traceutil/trace.go:172","msg":"trace[231539767] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-2; range_end:; response_count:1; response_revision:638; }","duration":"486.508741ms","start":"2026-03-26T14:51:34.021175Z","end":"2026-03-26T14:51:34.507684Z","steps":["trace[231539767] 'agreement among raft nodes before linearized reading' (duration: 486.341359ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.507732Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.021163Z","time spent":"486.562552ms","remote":"127.0.0.1:49434","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":1,"response size":8031,"request content":"key:\"/registry/pods/kube-system/kube-apiserver-controller-2\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:34.507734Z","caller":"traceutil/trace.go:172","msg":"trace[1699191485] range","detail":"{range_begin:/registry/endpointslices/default/kubernetes; range_end:; response_count:1; response_revision:638; }","duration":"486.42593ms","start":"2026-03-26T14:51:34.021292Z","end":"2026-03-26T14:51:34.507718Z","steps":["trace[1699191485] 'agreement among raft nodes before linearized reading' (duration: 486.192087ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.507774Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.021285Z","time spent":"486.480021ms","remote":"127.0.0.1:49570","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":521,"request content":"key:\"/registry/endpointslices/default/kubernetes\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:34.507626Z","caller":"traceutil/trace.go:172","msg":"trace[644157353] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:638; }","duration":"491.581015ms","start":"2026-03-26T14:51:34.016023Z","end":"2026-03-26T14:51:34.507604Z","steps":["trace[644157353] 'agreement among raft nodes before linearized reading' (duration: 491.319032ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.509996Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.016011Z","time spent":"493.971585ms","remote":"127.0.0.1:38928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:34.507493Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"264.182033ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-26T14:51:34.510183Z","caller":"traceutil/trace.go:172","msg":"trace[1982158110] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:638; }","duration":"266.867097ms","start":"2026-03-26T14:51:34.243302Z","end":"2026-03-26T14:51:34.510169Z","steps":["trace[1982158110] 'agreement among raft nodes before linearized reading' (duration: 264.108392ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.009312Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853143997,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:35.113751Z","caller":"traceutil/trace.go:172","msg":"trace[559788471] linearizableReadLoop","detail":"{readStateIndex:692; appliedIndex:692; }","duration":"604.806768ms","start":"2026-03-26T14:51:34.508927Z","end":"2026-03-26T14:51:35.113733Z","steps":["trace[559788471] 'read index received' (duration: 604.802358ms)","trace[559788471] 'applied index is now lower than readState.Index' (duration: 3.41µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:35.113910Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"604.97179ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.242\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-26T14:51:35.113938Z","caller":"traceutil/trace.go:172","msg":"trace[1683345641] range","detail":"{range_begin:/registry/masterleases/199.204.45.242; range_end:; response_count:1; response_revision:638; }","duration":"605.01311ms","start":"2026-03-26T14:51:34.508917Z","end":"2026-03-26T14:51:35.113930Z","steps":["trace[1683345641] 'agreement among raft nodes before linearized reading' (duration: 604.890359ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.113958Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.508905Z","time spent":"605.048721ms","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.242\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:35.454132Z","caller":"traceutil/trace.go:172","msg":"trace[555438780] transaction","detail":"{read_only:false; response_revision:641; number_of_response:1; }","duration":"936.141016ms","start":"2026-03-26T14:51:34.517970Z","end":"2026-03-26T14:51:35.454111Z","steps":["trace[555438780] 'process raft request' (duration: 936.093285ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:35.454149Z","caller":"traceutil/trace.go:172","msg":"trace[1826303465] transaction","detail":"{read_only:false; response_revision:640; number_of_response:1; }","duration":"937.157639ms","start":"2026-03-26T14:51:34.516964Z","end":"2026-03-26T14:51:35.454121Z","steps":["trace[1826303465] 'process raft request' (duration: 936.902906ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.454268Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.517950Z","time spent":"936.246537ms","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":426,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:35.454307Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.516939Z","time spent":"937.280491ms","remote":"127.0.0.1:49434","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7801,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:35.453796Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853144000,"received-request-id":1106650939853143997} {"level":"warn","ts":"2026-03-26T14:51:35.613912Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144000,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:35.818564Z","caller":"traceutil/trace.go:172","msg":"trace[1910482486] linearizableReadLoop","detail":"{readStateIndex:693; appliedIndex:695; }","duration":"704.747151ms","start":"2026-03-26T14:51:35.113799Z","end":"2026-03-26T14:51:35.818547Z","steps":["trace[1910482486] 'read index received' (duration: 704.742271ms)","trace[1910482486] 'applied index is now lower than readState.Index' (duration: 3.74µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:35.818692Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.270005274s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:35.818727Z","caller":"traceutil/trace.go:172","msg":"trace[855233767] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:641; }","duration":"1.270057205s","start":"2026-03-26T14:51:34.548661Z","end":"2026-03-26T14:51:35.818719Z","steps":["trace[855233767] 'agreement among raft nodes before linearized reading' (duration: 1.269982344s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.818751Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.30965442s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a06bb41491d8fb\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-26T14:51:35.818769Z","caller":"traceutil/trace.go:172","msg":"trace[493712147] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18a06bb41491d8fb; range_end:; response_count:1; response_revision:641; }","duration":"1.309677341s","start":"2026-03-26T14:51:34.509086Z","end":"2026-03-26T14:51:35.818763Z","steps":["trace[493712147] 'agreement among raft nodes before linearized reading' (duration: 1.309541919s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.818791Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.509063Z","time spent":"1.309722761s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":817,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a06bb41491d8fb\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:35.821455Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.114988Z","time spent":"706.462273ms","remote":"127.0.0.1:49216","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:36.249305Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853144003,"received-request-id":1106650939853144000} {"level":"warn","ts":"2026-03-26T14:51:36.319630Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144003,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:36.546232Z","caller":"traceutil/trace.go:172","msg":"trace[1166321337] linearizableReadLoop","detail":"{readStateIndex:696; appliedIndex:697; }","duration":"727.594229ms","start":"2026-03-26T14:51:35.818615Z","end":"2026-03-26T14:51:36.546209Z","steps":["trace[1166321337] 'read index received' (duration: 727.589739ms)","trace[1166321337] 'applied index is now lower than readState.Index' (duration: 3.42µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:36.546360Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.145129203s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:36.546469Z","caller":"traceutil/trace.go:172","msg":"trace[2112501965] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:642; }","duration":"1.145243965s","start":"2026-03-26T14:51:35.401203Z","end":"2026-03-26T14:51:36.546447Z","steps":["trace[2112501965] 'agreement among raft nodes before linearized reading' (duration: 1.145085223s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.546528Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.401182Z","time spent":"1.145333967s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:51:36.546545Z","caller":"traceutil/trace.go:172","msg":"trace[1572508816] transaction","detail":"{read_only:false; response_revision:644; number_of_response:1; }","duration":"724.007774ms","start":"2026-03-26T14:51:35.822518Z","end":"2026-03-26T14:51:36.546525Z","steps":["trace[1572508816] 'process raft request' (duration: 723.936013ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:36.546659Z","caller":"traceutil/trace.go:172","msg":"trace[268571274] transaction","detail":"{read_only:false; response_revision:643; number_of_response:1; }","duration":"726.254262ms","start":"2026-03-26T14:51:35.820387Z","end":"2026-03-26T14:51:36.546641Z","steps":["trace[268571274] 'process raft request' (duration: 725.910818ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.546678Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.822495Z","time spent":"724.117165ms","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:36.546778Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.820359Z","time spent":"726.343963ms","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:36.547899Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853144006,"received-request-id":1106650939853144003} {"level":"warn","ts":"2026-03-26T14:51:36.585425Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"765.658672ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:36.585586Z","caller":"traceutil/trace.go:172","msg":"trace[496068812] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:644; }","duration":"765.832394ms","start":"2026-03-26T14:51:35.819742Z","end":"2026-03-26T14:51:36.585574Z","steps":["trace[496068812] 'agreement among raft nodes before linearized reading' (duration: 765.624531ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.585503Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"748.214104ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:436"} {"level":"info","ts":"2026-03-26T14:51:36.585818Z","caller":"traceutil/trace.go:172","msg":"trace[616998953] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:644; }","duration":"748.529558ms","start":"2026-03-26T14:51:35.837272Z","end":"2026-03-26T14:51:36.585801Z","steps":["trace[616998953] 'agreement among raft nodes before linearized reading' (duration: 748.054622ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.585876Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.837261Z","time spent":"748.599818ms","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":459,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:36.702218Z","caller":"traceutil/trace.go:172","msg":"trace[422514479] linearizableReadLoop","detail":"{readStateIndex:700; appliedIndex:700; }","duration":"114.917019ms","start":"2026-03-26T14:51:36.587281Z","end":"2026-03-26T14:51:36.702198Z","steps":["trace[422514479] 'read index received' (duration: 114.911709ms)","trace[422514479] 'applied index is now lower than readState.Index' (duration: 3.83µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:36.702427Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"115.262104ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:36.702581Z","caller":"traceutil/trace.go:172","msg":"trace[242547163] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:645; }","duration":"115.510077ms","start":"2026-03-26T14:51:36.587055Z","end":"2026-03-26T14:51:36.702565Z","steps":["trace[242547163] 'agreement among raft nodes before linearized reading' (duration: 115.235954ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.202804Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144011,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:37.703259Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144011,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:37.772780Z","caller":"traceutil/trace.go:172","msg":"trace[479017644] transaction","detail":"{read_only:false; response_revision:647; number_of_response:1; }","duration":"1.183364562s","start":"2026-03-26T14:51:36.589375Z","end":"2026-03-26T14:51:37.772740Z","steps":["trace[479017644] 'process raft request' (duration: 1.18315763s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.772912Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.589322Z","time spent":"1.183523564s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":778,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:37.773002Z","caller":"traceutil/trace.go:172","msg":"trace[989294834] transaction","detail":"{read_only:false; response_revision:648; number_of_response:1; }","duration":"1.179280161s","start":"2026-03-26T14:51:36.593704Z","end":"2026-03-26T14:51:37.772984Z","steps":["trace[989294834] 'process raft request' (duration: 1.178976777s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.773128Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.593693Z","time spent":"1.179375382s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:37.773499Z","caller":"traceutil/trace.go:172","msg":"trace[1777665292] linearizableReadLoop","detail":"{readStateIndex:701; appliedIndex:704; }","duration":"1.071203816s","start":"2026-03-26T14:51:36.702280Z","end":"2026-03-26T14:51:37.773484Z","steps":["trace[1777665292] 'read index received' (duration: 1.071199056s)","trace[1777665292] 'applied index is now lower than readState.Index' (duration: 3.9µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:37.773927Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.599479Z","time spent":"1.17444286s","remote":"199.204.45.69:56366","response type":"/etcdserverpb.Cluster/MemberList","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:37.773971Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.1783887s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:37.774065Z","caller":"traceutil/trace.go:172","msg":"trace[1054466865] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:649; }","duration":"1.17844304s","start":"2026-03-26T14:51:36.595568Z","end":"2026-03-26T14:51:37.774011Z","steps":["trace[1054466865] 'agreement among raft nodes before linearized reading' (duration: 1.178353889s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.774109Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.595561Z","time spent":"1.178537362s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:37.774171Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.1848784s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:2 size:263"} {"level":"info","ts":"2026-03-26T14:51:37.774222Z","caller":"traceutil/trace.go:172","msg":"trace[1167836156] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:2; response_revision:649; }","duration":"1.184930731s","start":"2026-03-26T14:51:36.589277Z","end":"2026-03-26T14:51:37.774208Z","steps":["trace[1167836156] 'agreement among raft nodes before linearized reading' (duration: 1.184598507s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.774259Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.589260Z","time spent":"1.184989322s","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":2,"response size":286,"request content":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" "} {"level":"warn","ts":"2026-03-26T14:51:37.774523Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.146749936s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-26T14:51:37.774612Z","caller":"traceutil/trace.go:172","msg":"trace[72436619] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:649; }","duration":"1.146798587s","start":"2026-03-26T14:51:36.627755Z","end":"2026-03-26T14:51:37.774554Z","steps":["trace[72436619] 'agreement among raft nodes before linearized reading' (duration: 1.146013658s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.774745Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.627739Z","time spent":"1.146987299s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":533,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:38.274417Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144012,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:38.578790Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.030150387s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:38.578862Z","caller":"traceutil/trace.go:172","msg":"trace[1394263060] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:649; }","duration":"1.030236268s","start":"2026-03-26T14:51:37.548611Z","end":"2026-03-26T14:51:38.578847Z","steps":["trace[1394263060] 'agreement among raft nodes before linearized reading' (duration: 1.030127736s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:38.578984Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.12149959s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-26T14:51:38.579042Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"b59fd64328430f5b switched to configuration voters=(428247441089599436 13087414626088587099) learners=(6735209715263037861)"} {"level":"info","ts":"2026-03-26T14:51:38.579075Z","caller":"traceutil/trace.go:172","msg":"trace[1735143017] transaction","detail":"{read_only:false; response_revision:650; number_of_response:1; }","duration":"801.280741ms","start":"2026-03-26T14:51:37.777782Z","end":"2026-03-26T14:51:38.579063Z","steps":["trace[1735143017] 'process raft request' (duration: 801.002627ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:38.579072Z","caller":"traceutil/trace.go:172","msg":"trace[817663299] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:649; }","duration":"1.121601211s","start":"2026-03-26T14:51:37.457454Z","end":"2026-03-26T14:51:38.579055Z","steps":["trace[817663299] 'agreement among raft nodes before linearized reading' (duration: 1.121377327s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:38.579124Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"44a3176ec85feb9","local-member-id":"b59fd64328430f5b","added-peer-id":"5d78458d260361a5","added-peer-peer-urls":["https://199.204.45.69:2380"],"added-peer-is-learner":true} {"level":"warn","ts":"2026-03-26T14:51:38.579129Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.457435Z","time spent":"1.121682182s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":463,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:38.579156Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"5d78458d260361a5"} {"level":"warn","ts":"2026-03-26T14:51:38.579159Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.777760Z","time spent":"801.352563ms","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":739,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-26T14:51:38.579177Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.579327Z","caller":"traceutil/trace.go:172","msg":"trace[1572129871] linearizableReadLoop","detail":"{readStateIndex:704; appliedIndex:705; }","duration":"805.157669ms","start":"2026-03-26T14:51:37.773562Z","end":"2026-03-26T14:51:38.578719Z","steps":["trace[1572129871] 'read index received' (duration: 805.148429ms)","trace[1572129871] 'applied index is now lower than readState.Index' (duration: 5.78µs)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:38.580393Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.580938Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.581925Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.581966Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5","remote-peer-urls":["https://199.204.45.69:2380"]} {"level":"info","ts":"2026-03-26T14:51:38.581975Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.581992Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"warn","ts":"2026-03-26T14:51:38.643279Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"848.926198ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-26T14:51:38.643354Z","caller":"traceutil/trace.go:172","msg":"trace[278533131] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:652; }","duration":"849.016459ms","start":"2026-03-26T14:51:37.794323Z","end":"2026-03-26T14:51:38.643339Z","steps":["trace[278533131] 'agreement among raft nodes before linearized reading' (duration: 848.753396ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:38.643392Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.794304Z","time spent":"849.079341ms","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:38.643785Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"868.194297ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:38.643879Z","caller":"traceutil/trace.go:172","msg":"trace[332111924] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:653; }","duration":"868.286768ms","start":"2026-03-26T14:51:37.775559Z","end":"2026-03-26T14:51:38.643845Z","steps":["trace[332111924] 'agreement among raft nodes before linearized reading' (duration: 868.149777ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:38.643935Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.775540Z","time spent":"868.383869ms","remote":"127.0.0.1:38928","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:38.644038Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"867.902414ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:498"} {"level":"info","ts":"2026-03-26T14:51:38.644131Z","caller":"traceutil/trace.go:172","msg":"trace[1437162863] range","detail":"{range_begin:/registry/endpointslices/default/kubernetes; range_end:; response_count:1; response_revision:653; }","duration":"868.019295ms","start":"2026-03-26T14:51:37.776061Z","end":"2026-03-26T14:51:38.644080Z","steps":["trace[1437162863] 'agreement among raft nodes before linearized reading' (duration: 867.60388ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:38.644223Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.776047Z","time spent":"868.121546ms","remote":"127.0.0.1:49570","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":521,"request content":"key:\"/registry/endpointslices/default/kubernetes\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:38.709772Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"129.560598ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:38.709830Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"127.535453ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-2.18a06bb41491d8fb\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-26T14:51:38.709855Z","caller":"traceutil/trace.go:172","msg":"trace[1521667111] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:654; }","duration":"129.659009ms","start":"2026-03-26T14:51:38.580180Z","end":"2026-03-26T14:51:38.709839Z","steps":["trace[1521667111] 'agreement among raft nodes before linearized reading' (duration: 129.519008ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:38.709885Z","caller":"traceutil/trace.go:172","msg":"trace[1592559557] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-2.18a06bb41491d8fb; range_end:; response_count:1; response_revision:654; }","duration":"127.591204ms","start":"2026-03-26T14:51:38.582277Z","end":"2026-03-26T14:51:38.709868Z","steps":["trace[1592559557] 'agreement among raft nodes before linearized reading' (duration: 127.457012ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:39.436067Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.436118Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.436598Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.445968Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"b59fd64328430f5b","to":"5d78458d260361a5","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-26T14:51:39.446011Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.449832Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"b59fd64328430f5b","to":"5d78458d260361a5","stream-type":"stream Message"} {"level":"info","ts":"2026-03-26T14:51:39.449892Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"warn","ts":"2026-03-26T14:51:39.515929Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:51:39.515931Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5","error":"EOF"} {"level":"info","ts":"2026-03-26T14:51:39.530396Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.531009Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.542271Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"b59fd64328430f5b","to":"5d78458d260361a5","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-26T14:51:39.542309Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.542326Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.548051Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"b59fd64328430f5b","to":"5d78458d260361a5","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-26T14:51:39.548093Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.548111Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"b59fd64328430f5b","remote-peer-id":"5d78458d260361a5"} {"level":"warn","ts":"2026-03-26T14:51:39.762463Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.69:56406","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-26T14:51:39.765663Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"b59fd64328430f5b switched to configuration voters=(428247441089599436 6735209715263037861 13087414626088587099)"} {"level":"info","ts":"2026-03-26T14:51:39.765793Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"44a3176ec85feb9","local-member-id":"b59fd64328430f5b","promoted-member-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:42.416284Z","caller":"traceutil/trace.go:172","msg":"trace[1827147580] transaction","detail":"{read_only:false; response_revision:684; number_of_response:1; }","duration":"149.939472ms","start":"2026-03-26T14:51:42.266325Z","end":"2026-03-26T14:51:42.416265Z","steps":["trace[1827147580] 'process raft request' (duration: 149.75999ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:44.877616Z","caller":"traceutil/trace.go:172","msg":"trace[1477350547] transaction","detail":"{read_only:false; response_revision:688; number_of_response:1; }","duration":"139.292941ms","start":"2026-03-26T14:51:44.738308Z","end":"2026-03-26T14:51:44.877601Z","steps":["trace[1477350547] 'process raft request' (duration: 139.19881ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:45.795524Z","caller":"traceutil/trace.go:172","msg":"trace[949452842] transaction","detail":"{read_only:false; response_revision:690; number_of_response:1; }","duration":"117.912658ms","start":"2026-03-26T14:51:45.677586Z","end":"2026-03-26T14:51:45.795498Z","steps":["trace[949452842] 'process raft request' (duration: 117.712765ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:45.813099Z","caller":"traceutil/trace.go:172","msg":"trace[1776242307] transaction","detail":"{read_only:false; response_revision:692; number_of_response:1; }","duration":"126.356719ms","start":"2026-03-26T14:51:45.686728Z","end":"2026-03-26T14:51:45.813085Z","steps":["trace[1776242307] 'process raft request' (duration: 126.261108ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:45.813253Z","caller":"traceutil/trace.go:172","msg":"trace[1942268161] transaction","detail":"{read_only:false; response_revision:691; number_of_response:1; }","duration":"135.59834ms","start":"2026-03-26T14:51:45.677638Z","end":"2026-03-26T14:51:45.813236Z","steps":["trace[1942268161] 'process raft request' (duration: 135.253846ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:45.813256Z","caller":"traceutil/trace.go:172","msg":"trace[144166636] transaction","detail":"{read_only:false; response_revision:693; number_of_response:1; }","duration":"126.052026ms","start":"2026-03-26T14:51:45.687190Z","end":"2026-03-26T14:51:45.813242Z","steps":["trace[144166636] 'process raft request' (duration: 125.856713ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:46.014963Z","caller":"traceutil/trace.go:172","msg":"trace[2068645418] transaction","detail":"{read_only:false; response_revision:694; number_of_response:1; }","duration":"215.661602ms","start":"2026-03-26T14:51:45.799286Z","end":"2026-03-26T14:51:46.014948Z","steps":["trace[2068645418] 'process raft request' (duration: 215.563871ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:46.016136Z","caller":"traceutil/trace.go:172","msg":"trace[1107702212] linearizableReadLoop","detail":"{readStateIndex:754; appliedIndex:754; }","duration":"150.115554ms","start":"2026-03-26T14:51:45.865993Z","end":"2026-03-26T14:51:46.016109Z","steps":["trace[1107702212] 'read index received' (duration: 150.107394ms)","trace[1107702212] 'applied index is now lower than readState.Index' (duration: 6.6µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:46.016416Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"150.402247ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-26T14:51:46.016467Z","caller":"traceutil/trace.go:172","msg":"trace[145232441] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:694; }","duration":"150.473278ms","start":"2026-03-26T14:51:45.865981Z","end":"2026-03-26T14:51:46.016454Z","steps":["trace[145232441] 'agreement among raft nodes before linearized reading' (duration: 150.244355ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:46.147341Z","caller":"traceutil/trace.go:172","msg":"trace[746024802] transaction","detail":"{read_only:false; response_revision:696; number_of_response:1; }","duration":"126.130111ms","start":"2026-03-26T14:51:46.021193Z","end":"2026-03-26T14:51:46.147323Z","steps":["trace[746024802] 'process raft request' (duration: 122.760281ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:52.320994Z","caller":"traceutil/trace.go:172","msg":"trace[2121503637] transaction","detail":"{read_only:false; response_revision:723; number_of_response:1; }","duration":"141.870105ms","start":"2026-03-26T14:51:52.179102Z","end":"2026-03-26T14:51:52.320972Z","steps":["trace[2121503637] 'process raft request' (duration: 141.728093ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:52.420177Z","caller":"traceutil/trace.go:172","msg":"trace[1943010687] linearizableReadLoop","detail":"{readStateIndex:790; appliedIndex:791; }","duration":"239.102566ms","start":"2026-03-26T14:51:52.181061Z","end":"2026-03-26T14:51:52.420163Z","steps":["trace[1943010687] 'read index received' (duration: 239.098286ms)","trace[1943010687] 'applied index is now lower than readState.Index' (duration: 3.44µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:52.420469Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"239.40188ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:436"} {"level":"info","ts":"2026-03-26T14:51:52.420532Z","caller":"traceutil/trace.go:172","msg":"trace[1865176983] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:723; }","duration":"239.476841ms","start":"2026-03-26T14:51:52.181048Z","end":"2026-03-26T14:51:52.420524Z","steps":["trace[1865176983] 'agreement among raft nodes before linearized reading' (duration: 239.180346ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:52.549786Z","caller":"traceutil/trace.go:172","msg":"trace[1752995221] transaction","detail":"{read_only:false; response_revision:726; number_of_response:1; }","duration":"125.535664ms","start":"2026-03-26T14:51:52.424230Z","end":"2026-03-26T14:51:52.549765Z","steps":["trace[1752995221] 'process raft request' (duration: 125.378452ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:34.559765Z","caller":"traceutil/trace.go:172","msg":"trace[424203023] linearizableReadLoop","detail":"{readStateIndex:1158; appliedIndex:1158; }","duration":"127.132221ms","start":"2026-03-26T14:52:34.432607Z","end":"2026-03-26T14:52:34.559739Z","steps":["trace[424203023] 'read index received' (duration: 127.127421ms)","trace[424203023] 'applied index is now lower than readState.Index' (duration: 3.76µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:34.560125Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"127.465376ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/gateways.gateway.networking.k8s.io\" limit:1 ","response":"range_response_count:1 size:59622"} {"level":"info","ts":"2026-03-26T14:52:34.560331Z","caller":"traceutil/trace.go:172","msg":"trace[642502006] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions/gateways.gateway.networking.k8s.io; range_end:; response_count:1; response_revision:1003; }","duration":"127.706908ms","start":"2026-03-26T14:52:34.432603Z","end":"2026-03-26T14:52:34.560310Z","steps":["trace[642502006] 'agreement among raft nodes before linearized reading' (duration: 127.260844ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:34.560564Z","caller":"traceutil/trace.go:172","msg":"trace[436113532] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1004; }","duration":"125.420974ms","start":"2026-03-26T14:52:34.435127Z","end":"2026-03-26T14:52:34.560548Z","steps":["trace[436113532] 'process raft request' (duration: 125.05569ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.060200Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144655,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:52:35.209142Z","caller":"traceutil/trace.go:172","msg":"trace[1298248133] linearizableReadLoop","detail":"{readStateIndex:1163; appliedIndex:1164; }","duration":"649.28222ms","start":"2026-03-26T14:52:34.559844Z","end":"2026-03-26T14:52:35.209126Z","steps":["trace[1298248133] 'read index received' (duration: 649.27673ms)","trace[1298248133] 'applied index is now lower than readState.Index' (duration: 4.34µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:35.209320Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"660.364226ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:35.209342Z","caller":"traceutil/trace.go:172","msg":"trace[763434745] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1006; }","duration":"660.421577ms","start":"2026-03-26T14:52:34.548914Z","end":"2026-03-26T14:52:35.209336Z","steps":["trace[763434745] 'agreement among raft nodes before linearized reading' (duration: 660.340886ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:35.209590Z","caller":"traceutil/trace.go:172","msg":"trace[1780962244] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1007; }","duration":"619.631936ms","start":"2026-03-26T14:52:34.589940Z","end":"2026-03-26T14:52:35.209572Z","steps":["trace[1780962244] 'process raft request' (duration: 619.575976ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.209729Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.589918Z","time spent":"619.743107ms","remote":"127.0.0.1:49244","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":102,"response count":0,"response size":7742,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:52:35.210176Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"696.757361ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.242\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-26T14:52:35.210252Z","caller":"traceutil/trace.go:172","msg":"trace[426769692] range","detail":"{range_begin:/registry/masterleases/199.204.45.242; range_end:; response_count:1; response_revision:1006; }","duration":"696.979833ms","start":"2026-03-26T14:52:34.513256Z","end":"2026-03-26T14:52:35.210236Z","steps":["trace[426769692] 'agreement among raft nodes before linearized reading' (duration: 696.200105ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.210293Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.513238Z","time spent":"697.045953ms","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.242\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:35.222539Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853144658,"received-request-id":1106650939853144655} {"level":"info","ts":"2026-03-26T14:52:35.222849Z","caller":"traceutil/trace.go:172","msg":"trace[1901167027] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1007; }","duration":"631.011656ms","start":"2026-03-26T14:52:34.591821Z","end":"2026-03-26T14:52:35.222832Z","steps":["trace[1901167027] 'process raft request' (duration: 630.831624ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.223047Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.591802Z","time spent":"631.080497ms","remote":"127.0.0.1:49244","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":97,"response count":0,"response size":80730,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:52:35.224936Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"537.58334ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:35.224983Z","caller":"traceutil/trace.go:172","msg":"trace[1389565871] range","detail":"{range_begin:/registry/configmaps; range_end:; response_count:0; response_revision:1009; }","duration":"537.63732ms","start":"2026-03-26T14:52:34.687337Z","end":"2026-03-26T14:52:35.224974Z","steps":["trace[1389565871] 'agreement among raft nodes before linearized reading' (duration: 537.564309ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.225006Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.687317Z","time spent":"537.683451ms","remote":"127.0.0.1:49294","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/configmaps\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:35.225126Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"174.169676ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterroles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:35.225151Z","caller":"traceutil/trace.go:172","msg":"trace[754249699] range","detail":"{range_begin:/registry/clusterroles; range_end:; response_count:0; response_revision:1009; }","duration":"174.194246ms","start":"2026-03-26T14:52:35.050951Z","end":"2026-03-26T14:52:35.225145Z","steps":["trace[754249699] 'agreement among raft nodes before linearized reading' (duration: 174.157615ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.225307Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"244.450186ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-26T14:52:35.225330Z","caller":"traceutil/trace.go:172","msg":"trace[1649230290] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1009; }","duration":"244.473997ms","start":"2026-03-26T14:52:34.980851Z","end":"2026-03-26T14:52:35.225325Z","steps":["trace[1649230290] 'agreement among raft nodes before linearized reading' (duration: 244.387846ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.488593Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"128.615145ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:52:35.488697Z","caller":"traceutil/trace.go:172","msg":"trace[330795623] transaction","detail":"{read_only:false; response_revision:1018; number_of_response:1; }","duration":"134.292995ms","start":"2026-03-26T14:52:35.354389Z","end":"2026-03-26T14:52:35.488682Z","steps":["trace[330795623] 'compare' (duration: 128.475194ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:35.493665Z","caller":"traceutil/trace.go:172","msg":"trace[1783199289] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1018; }","duration":"130.909639ms","start":"2026-03-26T14:52:35.362729Z","end":"2026-03-26T14:52:35.493639Z","steps":["trace[1783199289] 'process raft request' (duration: 129.297162ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:35.509999Z","caller":"traceutil/trace.go:172","msg":"trace[276804915] linearizableReadLoop","detail":"{readStateIndex:1194; appliedIndex:1193; }","duration":"109.424163ms","start":"2026-03-26T14:52:35.400561Z","end":"2026-03-26T14:52:35.509985Z","steps":["trace[276804915] 'read index received' (duration: 109.255331ms)","trace[276804915] 'applied index is now lower than readState.Index' (duration: 168.342µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:35.510374Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"109.800547ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:35.510421Z","caller":"traceutil/trace.go:172","msg":"trace[1967128548] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1022; }","duration":"109.856707ms","start":"2026-03-26T14:52:35.400557Z","end":"2026-03-26T14:52:35.510414Z","steps":["trace[1967128548] 'agreement among raft nodes before linearized reading' (duration: 109.778417ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:36.075751Z","caller":"traceutil/trace.go:172","msg":"trace[873082254] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1028; }","duration":"275.379979ms","start":"2026-03-26T14:52:35.800357Z","end":"2026-03-26T14:52:36.075737Z","steps":["trace[873082254] 'process raft request' (duration: 274.980285ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:41.961818Z","caller":"traceutil/trace.go:172","msg":"trace[1374825607] linearizableReadLoop","detail":"{readStateIndex:1256; appliedIndex:1256; }","duration":"420.03666ms","start":"2026-03-26T14:52:41.541761Z","end":"2026-03-26T14:52:41.961798Z","steps":["trace[1374825607] 'read index received' (duration: 420.03093ms)","trace[1374825607] 'applied index is now lower than readState.Index' (duration: 4.6µs)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:52:41.962073Z","caller":"traceutil/trace.go:172","msg":"trace[790789579] transaction","detail":"{read_only:false; response_revision:1073; number_of_response:1; }","duration":"419.337902ms","start":"2026-03-26T14:52:41.542714Z","end":"2026-03-26T14:52:41.962052Z","steps":["trace[790789579] 'process raft request' (duration: 419.237121ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:41.962332Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:41.542693Z","time spent":"419.540926ms","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":696,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-26T14:52:41.967707Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"418.893078ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:41.967785Z","caller":"traceutil/trace.go:172","msg":"trace[845451930] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1073; }","duration":"418.980339ms","start":"2026-03-26T14:52:41.548789Z","end":"2026-03-26T14:52:41.967769Z","steps":["trace[845451930] 'agreement among raft nodes before linearized reading' (duration: 418.876927ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:41.967808Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"426.023953ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/flowschemas\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:41.967876Z","caller":"traceutil/trace.go:172","msg":"trace[1238140588] range","detail":"{range_begin:/registry/flowschemas; range_end:; response_count:0; response_revision:1073; }","duration":"426.109134ms","start":"2026-03-26T14:52:41.541749Z","end":"2026-03-26T14:52:41.967859Z","steps":["trace[1238140588] 'agreement among raft nodes before linearized reading' (duration: 420.261462ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:41.967916Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:41.541731Z","time spent":"426.175265ms","remote":"127.0.0.1:49858","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/flowschemas\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:41.967919Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"368.882567ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:52:41.967933Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"383.689891ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-03-26T14:52:41.967944Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"384.997485ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces/kube-system\" limit:1 ","response":"range_response_count:1 size:351"} {"level":"info","ts":"2026-03-26T14:52:41.967978Z","caller":"traceutil/trace.go:172","msg":"trace[753220990] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1073; }","duration":"368.947627ms","start":"2026-03-26T14:52:41.599017Z","end":"2026-03-26T14:52:41.967965Z","steps":["trace[753220990] 'agreement among raft nodes before linearized reading' (duration: 368.852406ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:41.967990Z","caller":"traceutil/trace.go:172","msg":"trace[1425228245] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1073; }","duration":"383.746721ms","start":"2026-03-26T14:52:41.584207Z","end":"2026-03-26T14:52:41.967975Z","steps":["trace[1425228245] 'agreement among raft nodes before linearized reading' (duration: 383.547509ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:41.968001Z","caller":"traceutil/trace.go:172","msg":"trace[1630934273] range","detail":"{range_begin:/registry/namespaces/kube-system; range_end:; response_count:1; response_revision:1073; }","duration":"385.056176ms","start":"2026-03-26T14:52:41.582930Z","end":"2026-03-26T14:52:41.967986Z","steps":["trace[1630934273] 'agreement among raft nodes before linearized reading' (duration: 384.904914ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:41.968024Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:41.584199Z","time spent":"383.815193ms","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:41.968032Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:41.582881Z","time spent":"385.143877ms","remote":"127.0.0.1:49318","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":1,"response size":374,"request content":"key:\"/registry/namespaces/kube-system\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:41.968012Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:41.599002Z","time spent":"369.003158ms","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:52:42.112647Z","caller":"traceutil/trace.go:172","msg":"trace[933875965] transaction","detail":"{read_only:false; response_revision:1074; number_of_response:1; }","duration":"139.916967ms","start":"2026-03-26T14:52:41.972710Z","end":"2026-03-26T14:52:42.112627Z","steps":["trace[933875965] 'process raft request' (duration: 139.813436ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:42.253713Z","caller":"traceutil/trace.go:172","msg":"trace[1346824390] linearizableReadLoop","detail":"{readStateIndex:1259; appliedIndex:1260; }","duration":"279.968355ms","start":"2026-03-26T14:52:41.973708Z","end":"2026-03-26T14:52:42.253676Z","steps":["trace[1346824390] 'read index received' (duration: 279.961485ms)","trace[1346824390] 'applied index is now lower than readState.Index' (duration: 5.57µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:42.253978Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"280.239618ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/kube-system/cilium-config\" limit:1 ","response":"range_response_count:1 size:6534"} {"level":"info","ts":"2026-03-26T14:52:42.254086Z","caller":"traceutil/trace.go:172","msg":"trace[1828988070] range","detail":"{range_begin:/registry/configmaps/kube-system/cilium-config; range_end:; response_count:1; response_revision:1074; }","duration":"280.365269ms","start":"2026-03-26T14:52:41.973703Z","end":"2026-03-26T14:52:42.254068Z","steps":["trace[1828988070] 'agreement among raft nodes before linearized reading' (duration: 280.089726ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:42.254122Z","caller":"traceutil/trace.go:172","msg":"trace[1870172899] transaction","detail":"{read_only:false; response_revision:1076; number_of_response:1; }","duration":"271.130513ms","start":"2026-03-26T14:52:41.982968Z","end":"2026-03-26T14:52:42.254099Z","steps":["trace[1870172899] 'process raft request' (duration: 270.928601ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:43.784725Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144732,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:52:44.256187Z","caller":"traceutil/trace.go:172","msg":"trace[630631298] linearizableReadLoop","detail":"{readStateIndex:1266; appliedIndex:1266; }","duration":"971.80239ms","start":"2026-03-26T14:52:43.284363Z","end":"2026-03-26T14:52:44.256166Z","steps":["trace[630631298] 'read index received' (duration: 971.79799ms)","trace[630631298] 'applied index is now lower than readState.Index' (duration: 3.34µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:44.256426Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"972.054193ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-26T14:52:44.256451Z","caller":"traceutil/trace.go:172","msg":"trace[949074961] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1080; }","duration":"972.094384ms","start":"2026-03-26T14:52:43.284349Z","end":"2026-03-26T14:52:44.256443Z","steps":["trace[949074961] 'agreement among raft nodes before linearized reading' (duration: 971.964581ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:44.256486Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.284332Z","time spent":"972.148565ms","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:44.756430Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144736,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:52:45.257251Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1106650939853144736,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:52:45.398656Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000041817s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:52:45.398954Z","caller":"traceutil/trace.go:172","msg":"trace[1819389652] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000351671s","start":"2026-03-26T14:52:43.398588Z","end":"2026-03-26T14:52:45.398940Z","steps":["trace[1819389652] 'agreement among raft nodes before linearized reading' (duration: 2.000038747s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.399031Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.398572Z","time spent":"2.000450101s","remote":"127.0.0.1:38922","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:52:45 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:52:45.619322Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1233","msg":"b59fd64328430f5b [logterm: 2, index: 1271, vote: 0] cast MsgPreVote for 5d78458d260361a5 [logterm: 2, index: 1271] at term 2"} {"level":"info","ts":"2026-03-26T14:52:45.621182Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"b59fd64328430f5b [term: 2] received a MsgVote message with higher term from 5d78458d260361a5 [term: 3]"} {"level":"info","ts":"2026-03-26T14:52:45.621213Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"b59fd64328430f5b became follower at term 3"} {"level":"info","ts":"2026-03-26T14:52:45.621224Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1233","msg":"b59fd64328430f5b [logterm: 2, index: 1271, vote: 0] cast MsgVote for 5d78458d260361a5 [logterm: 2, index: 1271] at term 3"} {"level":"info","ts":"2026-03-26T14:52:45.621235Z","logger":"raft","caller":"v3@v3.6.0/node.go:376","msg":"raft.node: b59fd64328430f5b lost leader 5f170c75a1d97cc at term 3"} {"level":"info","ts":"2026-03-26T14:52:45.622821Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: b59fd64328430f5b elected leader 5d78458d260361a5 at term 3"} {"level":"warn","ts":"2026-03-26T14:52:45.622941Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.506449473s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-26T14:52:45.622973Z","caller":"traceutil/trace.go:172","msg":"trace[158925699] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; }","duration":"1.506500704s","start":"2026-03-26T14:52:44.116463Z","end":"2026-03-26T14:52:45.622963Z","steps":["trace[158925699] 'agreement among raft nodes before linearized reading' (duration: 1.506448393s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.623002Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.116446Z","time spent":"1.506550294s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:45.623037Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.674583348s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-29xw4\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-26T14:52:45.623047Z","caller":"traceutil/trace.go:172","msg":"trace[1211671418] range","detail":"{range_begin:/registry/pods/kube-system/cilium-29xw4; range_end:; }","duration":"1.674595738s","start":"2026-03-26T14:52:43.948448Z","end":"2026-03-26T14:52:45.623044Z","steps":["trace[1211671418] 'agreement among raft nodes before linearized reading' (duration: 1.674583608s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.623061Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.948431Z","time spent":"1.674626228s","remote":"127.0.0.1:49434","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":0,"response size":0,"request content":"key:\"/registry/pods/kube-system/cilium-29xw4\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:45.623081Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.073800196s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-26T14:52:45.623094Z","caller":"traceutil/trace.go:172","msg":"trace[930328859] range","detail":"{range_begin:; range_end:; }","duration":"2.073814466s","start":"2026-03-26T14:52:43.549276Z","end":"2026-03-26T14:52:45.623090Z","steps":["trace[930328859] 'agreement among raft nodes before linearized reading' (duration: 2.073800126s)"],"step_count":1} {"level":"error","ts":"2026-03-26T14:52:45.623137Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[+]non_learner ok\n[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: leader changed\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"info","ts":"2026-03-26T14:52:45.624885Z","caller":"traceutil/trace.go:172","msg":"trace[447014104] transaction","detail":"{read_only:false; response_revision:1083; number_of_response:1; }","duration":"1.675531469s","start":"2026-03-26T14:52:43.949346Z","end":"2026-03-26T14:52:45.624878Z","steps":["trace[447014104] 'process raft request' (duration: 1.675455738s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.624943Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.949334Z","time spent":"1.675575779s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":751,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-26T14:52:45.625051Z","caller":"traceutil/trace.go:172","msg":"trace[2103438198] transaction","detail":"{read_only:false; response_revision:1084; number_of_response:1; }","duration":"1.465492848s","start":"2026-03-26T14:52:44.159551Z","end":"2026-03-26T14:52:45.625044Z","steps":["trace[2103438198] 'process raft request' (duration: 1.465274446s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.625086Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.159522Z","time spent":"1.465541529s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":676,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:52:45.625147Z","caller":"traceutil/trace.go:172","msg":"trace[1685649346] transaction","detail":"{read_only:false; response_revision:1085; number_of_response:1; }","duration":"1.445725024s","start":"2026-03-26T14:52:44.179415Z","end":"2026-03-26T14:52:45.625140Z","steps":["trace[1685649346] 'process raft request' (duration: 1.445431821s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:45.625111Z","caller":"etcdserver/v3_server.go:903","msg":"first commit in current term: resending ReadIndex request"} {"level":"warn","ts":"2026-03-26T14:52:45.625177Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.179374Z","time spent":"1.445786014s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:52:45.625693Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"226.657457ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:45.625867Z","caller":"traceutil/trace.go:172","msg":"trace[1136588410] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1085; }","duration":"226.846539ms","start":"2026-03-26T14:52:45.398991Z","end":"2026-03-26T14:52:45.625838Z","steps":["trace[1136588410] 'agreement among raft nodes before linearized reading' (duration: 226.643087ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.626174Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.112327576s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.242\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-26T14:52:45.626218Z","caller":"traceutil/trace.go:172","msg":"trace[1112670765] range","detail":"{range_begin:/registry/masterleases/199.204.45.242; range_end:; response_count:1; response_revision:1085; }","duration":"1.112372067s","start":"2026-03-26T14:52:44.513840Z","end":"2026-03-26T14:52:45.626212Z","steps":["trace[1112670765] 'agreement among raft nodes before linearized reading' (duration: 1.112150445s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.626234Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.513825Z","time spent":"1.112404147s","remote":"127.0.0.1:49216","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.242\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:45.626505Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":1106650939853144741,"received-request-id":1106650939853144740} {"level":"warn","ts":"2026-03-26T14:52:45.626300Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"287.047593ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ipaddresses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:52:45.627462Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.157445914s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:110 size:9396"} {"level":"info","ts":"2026-03-26T14:52:45.627513Z","caller":"traceutil/trace.go:172","msg":"trace[1485225562] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:110; response_revision:1085; }","duration":"1.157499495s","start":"2026-03-26T14:52:44.470002Z","end":"2026-03-26T14:52:45.627501Z","steps":["trace[1485225562] 'agreement among raft nodes before linearized reading' (duration: 1.156257082s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.627548Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.469981Z","time spent":"1.157553605s","remote":"127.0.0.1:49268","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":110,"response size":9419,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"info","ts":"2026-03-26T14:52:45.627640Z","caller":"traceutil/trace.go:172","msg":"trace[1134676731] range","detail":"{range_begin:/registry/ipaddresses; range_end:; response_count:0; response_revision:1085; }","duration":"287.394057ms","start":"2026-03-26T14:52:45.339231Z","end":"2026-03-26T14:52:45.626625Z","steps":["trace[1134676731] 'agreement among raft nodes before linearized reading' (duration: 286.951831ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:45.971851Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"b59fd64328430f5b [term: 3] ignored a MsgReadIndexResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"warn","ts":"2026-03-26T14:52:49.285791Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38910","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.289304Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.290188Z","time spent":"4.999113722s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} 2026/03/26 14:52:49 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:52:49.309554Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38916","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.354408Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38930","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.466586Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38944","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.478268Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.480076Z","time spent":"4.998185399s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} 2026/03/26 14:52:49 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:52:49.490422Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38946","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.509529Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38958","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.565035Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38966","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.574237Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38970","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.622506Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38992","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.651467Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39012","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.667628Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39030","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.695288Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39040","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.711594Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39052","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.732002Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39074","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.751060Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39106","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.838655Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39110","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:49.907751Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39132","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:50.031028Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39146","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:50.054009Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39166","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:50.077780Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39182","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:50.096089Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39216","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:50.120338Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39242","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:50.134237Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39246","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:50.153497Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39260","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:50.164311Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39282","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:51.261048Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.260359Z","time spent":"7.000684019s","remote":"127.0.0.1:49548","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:52:54.196101Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39308","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:57.809139Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39640","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:57.819965Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39664","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:57.956767Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39682","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:58.104178Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39696","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:52:58.210895Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39714","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-26T14:53:05.820055Z","caller":"traceutil/trace.go:172","msg":"trace[930698517] transaction","detail":"{read_only:false; response_revision:1381; number_of_response:1; }","duration":"265.112389ms","start":"2026-03-26T14:53:05.554921Z","end":"2026-03-26T14:53:05.820033Z","steps":["trace[930698517] 'process raft request' (duration: 264.969968ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:53:21.192806Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:35628","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:53:21.207374Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:35652","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:53:21.238289Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:35668","server-name":"","error":"EOF"}