{"level":"warn","ts":"2026-03-30T21:09:07.134447Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"warn","ts":"2026-03-30T21:09:07.134839Z","caller":"etcdmain/config.go:270","msg":"--snapshot-count is deprecated in 3.6 and will be decommissioned in 3.7."} {"level":"info","ts":"2026-03-30T21:09:07.134879Z","caller":"etcdmain/etcd.go:64","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.204.45.74:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--feature-gates=InitialCorruptCheck=true","--initial-advertise-peer-urls=https://199.204.45.74:2380","--initial-cluster=controller-1=https://199.204.45.74:2380","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.204.45.74:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.204.45.74:2380","--name=controller-1","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--watch-progress-notify-interval=5s"]} {"level":"info","ts":"2026-03-30T21:09:07.134995Z","caller":"etcdmain/etcd.go:124","msg":"Initialize and start etcd server","data-dir":"/var/lib/etcd","dir-type":"empty"} {"level":"warn","ts":"2026-03-30T21:09:07.135093Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-30T21:09:07.135120Z","caller":"embed/etcd.go:138","msg":"configuring peer listeners","listen-peer-urls":["https://199.204.45.74:2380"]} {"level":"info","ts":"2026-03-30T21:09:07.135151Z","caller":"embed/etcd.go:544","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-30T21:09:07.135840Z","caller":"embed/etcd.go:146","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.74:2379"]} {"level":"info","ts":"2026-03-30T21:09:07.135999Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.6.4","git-sha":"5400cdc","go-version":"go1.23.11","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-1","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.204.45.74:2380"],"listen-peer-urls":["https://199.204.45.74:2380"],"advertise-client-urls":["https://199.204.45.74:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.74:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"experimental-local-address":"","cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://199.204.45.74:2380","initial-cluster-state":"new","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"feature-gates":"InitialCorruptCheck=true","initial-corrupt-check":false,"corrupt-check-time-interval":"0s","compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","discovery-token":"","discovery-endpoints":"","discovery-dial-timeout":"2s","discovery-request-timeout":"5s","discovery-keepalive-time":"2s","discovery-keepalive-timeout":"6s","discovery-insecure-transport":true,"discovery-insecure-skip-tls-verify":false,"discovery-cert":"","discovery-key":"","discovery-cacert":"","discovery-user":"","downgrade-check-interval":"5s","max-learners":1,"v2-deprecation":"write-only"} {"level":"info","ts":"2026-03-30T21:09:07.137395Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc0004c6180}"} {"level":"info","ts":"2026-03-30T21:09:07.146028Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-30T21:09:07.146227Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"9.114585ms"} {"level":"info","ts":"2026-03-30T21:09:07.146289Z","caller":"etcdserver/bootstrap.go:220","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-03-30T21:09:07.146309Z","caller":"etcdserver/bootstrap.go:94","msg":"bootstrapping cluster"} {"level":"info","ts":"2026-03-30T21:09:07.146357Z","caller":"etcdserver/bootstrap.go:101","msg":"bootstrapping storage"} {"level":"info","ts":"2026-03-30T21:09:07.159343Z","caller":"etcdserver/bootstrap.go:109","msg":"bootstrapping raft"} {"level":"info","ts":"2026-03-30T21:09:07.159454Z","caller":"etcdserver/bootstrap.go:499","msg":"starting local member","local-member-id":"89ba38f964b95aa0","cluster-id":"54492f71bf65c400"} {"level":"info","ts":"2026-03-30T21:09:07.159483Z","caller":"etcdserver/server.go:312","msg":"bootstrap successfully"} {"level":"info","ts":"2026-03-30T21:09:07.159554Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"89ba38f964b95aa0 switched to configuration voters=()"} {"level":"info","ts":"2026-03-30T21:09:07.159594Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"89ba38f964b95aa0 became follower at term 0"} {"level":"info","ts":"2026-03-30T21:09:07.159602Z","logger":"raft","caller":"v3@v3.6.0/raft.go:493","msg":"newRaft 89ba38f964b95aa0 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"info","ts":"2026-03-30T21:09:07.159615Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"89ba38f964b95aa0 became follower at term 1"} {"level":"info","ts":"2026-03-30T21:09:07.159643Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"89ba38f964b95aa0 switched to configuration voters=(9924307372652190368)"} {"level":"warn","ts":"2026-03-30T21:09:07.167963Z","caller":"auth/store.go:1135","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-30T21:09:07.170341Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-30T21:09:07.174413Z","caller":"storage/quota.go:93","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-30T21:09:07.174913Z","caller":"etcdserver/server.go:598","msg":"starting etcd server","local-member-id":"89ba38f964b95aa0","local-server-version":"3.6.4","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-30T21:09:07.175032Z","caller":"etcdserver/server.go:483","msg":"started as single-node; fast-forwarding election ticks","local-member-id":"89ba38f964b95aa0","forward-ticks":9,"forward-duration":"900ms","election-ticks":10,"election-timeout":"1s"} {"level":"info","ts":"2026-03-30T21:09:07.175160Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-30T21:09:07.175279Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-30T21:09:07.175298Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-30T21:09:07.175310Z","caller":"embed/etcd.go:766","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-30T21:09:07.175408Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"89ba38f964b95aa0 switched to configuration voters=(9924307372652190368)"} {"level":"info","ts":"2026-03-30T21:09:07.175535Z","caller":"embed/etcd.go:640","msg":"serving peer traffic","address":"199.204.45.74:2380"} {"level":"info","ts":"2026-03-30T21:09:07.175576Z","caller":"embed/etcd.go:611","msg":"cmux::serve","address":"199.204.45.74:2380"} {"level":"info","ts":"2026-03-30T21:09:07.175590Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"54492f71bf65c400","local-member-id":"89ba38f964b95aa0","added-peer-id":"89ba38f964b95aa0","added-peer-peer-urls":["https://199.204.45.74:2380"],"added-peer-is-learner":false} {"level":"info","ts":"2026-03-30T21:09:07.175865Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"89ba38f964b95aa0","initial-advertise-peer-urls":["https://199.204.45.74:2380"],"listen-peer-urls":["https://199.204.45.74:2380"],"advertise-client-urls":["https://199.204.45.74:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.74:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-30T21:09:07.175954Z","caller":"embed/etcd.go:890","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-30T21:09:08.060143Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"89ba38f964b95aa0 is starting a new election at term 1"} {"level":"info","ts":"2026-03-30T21:09:08.060195Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"89ba38f964b95aa0 became pre-candidate at term 1"} {"level":"info","ts":"2026-03-30T21:09:08.060252Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"89ba38f964b95aa0 received MsgPreVoteResp from 89ba38f964b95aa0 at term 1"} {"level":"info","ts":"2026-03-30T21:09:08.060263Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"89ba38f964b95aa0 has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T21:09:08.060278Z","logger":"raft","caller":"v3@v3.6.0/raft.go:912","msg":"89ba38f964b95aa0 became candidate at term 2"} {"level":"info","ts":"2026-03-30T21:09:08.061080Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"89ba38f964b95aa0 received MsgVoteResp from 89ba38f964b95aa0 at term 2"} {"level":"info","ts":"2026-03-30T21:09:08.061153Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"89ba38f964b95aa0 has received 1 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-30T21:09:08.061204Z","logger":"raft","caller":"v3@v3.6.0/raft.go:970","msg":"89ba38f964b95aa0 became leader at term 2"} {"level":"info","ts":"2026-03-30T21:09:08.061231Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: 89ba38f964b95aa0 elected leader 89ba38f964b95aa0 at term 2"} {"level":"info","ts":"2026-03-30T21:09:08.062127Z","caller":"etcdserver/server.go:1804","msg":"published local member to cluster through raft","local-member-id":"89ba38f964b95aa0","local-member-attributes":"{Name:controller-1 ClientURLs:[https://199.204.45.74:2379]}","cluster-id":"54492f71bf65c400","publish-timeout":"7s"} {"level":"info","ts":"2026-03-30T21:09:08.062155Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-30T21:09:08.062269Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-30T21:09:08.062531Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-30T21:09:08.062572Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-03-30T21:09:08.063215Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-30T21:09:08.065071Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"warn","ts":"2026-03-30T21:09:08.065646Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45436","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.065747Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45416","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.066657Z","caller":"v3rpc/grpc.go:52","msg":"etcdserver: failed to register grpc metrics","error":"duplicate metrics collector registration attempted"} {"level":"info","ts":"2026-03-30T21:09:08.066782Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-30T21:09:08.062228Z","caller":"etcdserver/server.go:2404","msg":"setting up initial cluster version using v3 API","cluster-version":"3.6"} {"level":"info","ts":"2026-03-30T21:09:08.069679Z","caller":"membership/cluster.go:682","msg":"set initial cluster version","cluster-id":"54492f71bf65c400","local-member-id":"89ba38f964b95aa0","cluster-version":"3.6"} {"level":"info","ts":"2026-03-30T21:09:08.070429Z","caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.6"} {"level":"info","ts":"2026-03-30T21:09:08.073571Z","caller":"etcdserver/server.go:2424","msg":"cluster version is updated","cluster-version":"3.6"} {"level":"info","ts":"2026-03-30T21:09:08.072506Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.204.45.74:2379"} {"level":"info","ts":"2026-03-30T21:09:08.073779Z","caller":"version/monitor.go:116","msg":"cluster version differs from storage version.","cluster-version":"3.6.0","storage-version":"3.5.0"} {"level":"info","ts":"2026-03-30T21:09:08.076897Z","caller":"schema/migration.go:65","msg":"updated storage version","new-storage-version":"3.6.0"} {"level":"warn","ts":"2026-03-30T21:09:08.082197Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45466","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.094799Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45506","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.117674Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45512","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.127625Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45536","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.136950Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45540","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.156516Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45584","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.166153Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45604","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.176709Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45614","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.185175Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45642","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.193385Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45660","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.200797Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45668","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.208279Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45690","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.224778Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45716","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.233436Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45718","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.257367Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45762","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.261893Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45776","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.274495Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45792","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.304041Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45814","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.316502Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45820","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.324808Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45830","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.331520Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45850","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.342807Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45856","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.357981Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45868","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.367802Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45878","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.375104Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45898","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.396068Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45904","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.414762Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45938","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.423976Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45950","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.431744Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45958","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.439451Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45968","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.447847Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45976","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.455149Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:45990","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.462831Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46030","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.476132Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46044","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.486668Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46078","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.496150Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46088","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.504952Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46106","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.516872Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46130","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.524359Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46140","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.533088Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46156","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.556322Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46174","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.575853Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46204","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.585857Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46210","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.595576Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46228","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.602882Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46246","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.615611Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46258","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.623281Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46268","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.633749Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46286","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.645837Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46302","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.665734Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46334","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.701146Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46354","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.718922Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46388","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:08.779644Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46402","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:09:16.147415Z","caller":"traceutil/trace.go:172","msg":"trace[1244125411] transaction","detail":"{read_only:false; response_revision:355; number_of_response:1; }","duration":"145.695673ms","start":"2026-03-30T21:09:16.001694Z","end":"2026-03-30T21:09:16.147390Z","steps":["trace[1244125411] 'process raft request' (duration: 143.93942ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:16.733884Z","caller":"traceutil/trace.go:172","msg":"trace[1806661294] transaction","detail":"{read_only:false; response_revision:361; number_of_response:1; }","duration":"101.145602ms","start":"2026-03-30T21:09:16.632716Z","end":"2026-03-30T21:09:16.733862Z","steps":["trace[1806661294] 'process raft request' (duration: 100.468583ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:16.733943Z","caller":"traceutil/trace.go:172","msg":"trace[1819921835] transaction","detail":"{read_only:false; response_revision:362; number_of_response:1; }","duration":"100.751297ms","start":"2026-03-30T21:09:16.633167Z","end":"2026-03-30T21:09:16.733918Z","steps":["trace[1819921835] 'process raft request' (duration: 100.396412ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:23.906894Z","caller":"traceutil/trace.go:172","msg":"trace[1164014922] transaction","detail":"{read_only:false; response_revision:430; number_of_response:1; }","duration":"131.996205ms","start":"2026-03-30T21:09:23.774884Z","end":"2026-03-30T21:09:23.906881Z","steps":["trace[1164014922] 'process raft request' (duration: 131.903223ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:09:24.563878Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"168.655734ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:09:24.564852Z","caller":"traceutil/trace.go:172","msg":"trace[1492744691] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:434; }","duration":"169.628738ms","start":"2026-03-30T21:09:24.395190Z","end":"2026-03-30T21:09:24.564819Z","steps":["trace[1492744691] 'range keys from in-memory index tree' (duration: 168.443633ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:28.317019Z","caller":"traceutil/trace.go:172","msg":"trace[1545998650] transaction","detail":"{read_only:false; response_revision:444; number_of_response:1; }","duration":"101.925878ms","start":"2026-03-30T21:09:28.215042Z","end":"2026-03-30T21:09:28.316968Z","steps":["trace[1545998650] 'process raft request' (duration: 101.855947ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:30.804004Z","caller":"traceutil/trace.go:172","msg":"trace[516414361] transaction","detail":"{read_only:false; response_revision:450; number_of_response:1; }","duration":"170.530777ms","start":"2026-03-30T21:09:30.633455Z","end":"2026-03-30T21:09:30.803986Z","steps":["trace[516414361] 'process raft request' (duration: 170.402135ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:09:31.688422Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"484.979207ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:09:31.688518Z","caller":"traceutil/trace.go:172","msg":"trace[2099792159] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:450; }","duration":"485.091209ms","start":"2026-03-30T21:09:31.203404Z","end":"2026-03-30T21:09:31.688495Z","steps":["trace[2099792159] 'range keys from in-memory index tree' (duration: 484.914447ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:09:31.688573Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"851.477911ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.74\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-30T21:09:31.688656Z","caller":"traceutil/trace.go:172","msg":"trace[1653617626] range","detail":"{range_begin:/registry/masterleases/199.204.45.74; range_end:; response_count:1; response_revision:450; }","duration":"851.580462ms","start":"2026-03-30T21:09:30.837059Z","end":"2026-03-30T21:09:31.688639Z","steps":["trace[1653617626] 'range keys from in-memory index tree' (duration: 851.299128ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:09:31.688698Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:09:30.837036Z","time spent":"851.651772ms","remote":"127.0.0.1:45480","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":1,"response size":156,"request content":"key:\"/registry/masterleases/199.204.45.74\" limit:1 "} {"level":"info","ts":"2026-03-30T21:09:32.516773Z","caller":"traceutil/trace.go:172","msg":"trace[1668400656] linearizableReadLoop","detail":"{readStateIndex:469; appliedIndex:469; }","duration":"409.729767ms","start":"2026-03-30T21:09:32.107014Z","end":"2026-03-30T21:09:32.516744Z","steps":["trace[1668400656] 'read index received' (duration: 409.720457ms)","trace[1668400656] 'applied index is now lower than readState.Index' (duration: 7.79µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:09:32.516931Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"409.900821ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:09:32.517016Z","caller":"traceutil/trace.go:172","msg":"trace[2114960375] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:450; }","duration":"409.998802ms","start":"2026-03-30T21:09:32.106999Z","end":"2026-03-30T21:09:32.516998Z","steps":["trace[2114960375] 'agreement among raft nodes before linearized reading' (duration: 409.84372ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:09:32.517029Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:09:31.690024Z","time spent":"827.0008ms","remote":"127.0.0.1:45480","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-30T21:09:32.517063Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:09:32.106973Z","time spent":"410.078602ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:09:32.559905Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"357.498094ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:09:32.559991Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"227.650424ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:09:32.559999Z","caller":"traceutil/trace.go:172","msg":"trace[1449606719] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:450; }","duration":"357.603125ms","start":"2026-03-30T21:09:32.202375Z","end":"2026-03-30T21:09:32.559978Z","steps":["trace[1449606719] 'agreement among raft nodes before linearized reading' (duration: 357.459703ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:32.560044Z","caller":"traceutil/trace.go:172","msg":"trace[1240406205] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:450; }","duration":"227.733536ms","start":"2026-03-30T21:09:32.332296Z","end":"2026-03-30T21:09:32.560030Z","steps":["trace[1240406205] 'agreement among raft nodes before linearized reading' (duration: 227.541043ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:32.562387Z","caller":"traceutil/trace.go:172","msg":"trace[2033799026] transaction","detail":"{read_only:false; response_revision:451; number_of_response:1; }","duration":"607.918119ms","start":"2026-03-30T21:09:31.954446Z","end":"2026-03-30T21:09:32.562364Z","steps":["trace[2033799026] 'process raft request' (duration: 605.606589ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:32.562584Z","caller":"traceutil/trace.go:172","msg":"trace[702331728] transaction","detail":"{read_only:false; response_revision:452; number_of_response:1; }","duration":"604.441645ms","start":"2026-03-30T21:09:31.958124Z","end":"2026-03-30T21:09:32.562565Z","steps":["trace[702331728] 'process raft request' (duration: 603.754786ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:09:32.563053Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:09:31.958108Z","time spent":"604.513726ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:09:32.563201Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:09:31.954417Z","time spent":"608.03439ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:09:42.876998Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"173.803539ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:09:42.877078Z","caller":"traceutil/trace.go:172","msg":"trace[1239741905] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:477; }","duration":"173.9021ms","start":"2026-03-30T21:09:42.703161Z","end":"2026-03-30T21:09:42.877063Z","steps":["trace[1239741905] 'range keys from in-memory index tree' (duration: 173.746759ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:09:44.807565Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"89ba38f964b95aa0 switched to configuration voters=(9924307372652190368) learners=(8910420216259651266)"} {"level":"info","ts":"2026-03-30T21:09:44.807759Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"54492f71bf65c400","local-member-id":"89ba38f964b95aa0","added-peer-id":"7ba82bfa2d9da6c2","added-peer-peer-urls":["https://199.204.45.38:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-30T21:09:44.807819Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:44.807845Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:44.808029Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:44.808098Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:44.808125Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:44.808156Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:44.808143Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:44.808154Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2","remote-peer-urls":["https://199.204.45.38:2380"]} {"level":"info","ts":"2026-03-30T21:09:44.808324Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"89ba38f964b95aa0","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"7ba82bfa2d9da6c2"} {"level":"error","ts":"2026-03-30T21:09:44.853153Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-30T21:09:45.359825Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-30T21:09:45.855884Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-30T21:09:46.357253Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"warn","ts":"2026-03-30T21:09:46.882837Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"14.696004ms"} {"level":"info","ts":"2026-03-30T21:09:46.884137Z","caller":"traceutil/trace.go:172","msg":"trace[1890554482] transaction","detail":"{read_only:false; response_revision:486; number_of_response:1; }","duration":"211.770219ms","start":"2026-03-30T21:09:46.672283Z","end":"2026-03-30T21:09:46.884054Z","steps":["trace[1890554482] 'process raft request' (duration: 210.640874ms)"],"step_count":1} {"level":"error","ts":"2026-03-30T21:09:46.885219Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-30T21:09:47.358423Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-30T21:09:47.856107Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"warn","ts":"2026-03-30T21:09:48.069393Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://199.204.45.38:2380/version","remote-member-id":"7ba82bfa2d9da6c2","error":"Get \"https://199.204.45.38:2380/version\": dial tcp 199.204.45.38:2380: connect: connection refused"} {"level":"warn","ts":"2026-03-30T21:09:48.069520Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"7ba82bfa2d9da6c2","error":"Get \"https://199.204.45.38:2380/version\": dial tcp 199.204.45.38:2380: connect: connection refused"} {"level":"info","ts":"2026-03-30T21:09:48.241923Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.242005Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.242700Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.251655Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"89ba38f964b95aa0","to":"7ba82bfa2d9da6c2","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-30T21:09:48.251681Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.252008Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"89ba38f964b95aa0","to":"7ba82bfa2d9da6c2","stream-type":"stream Message"} {"level":"info","ts":"2026-03-30T21:09:48.252215Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.268393Z","caller":"etcdserver/server.go:1838","msg":"sending merged snapshot","from":"89ba38f964b95aa0","to":"7ba82bfa2d9da6c2","bytes":815105,"size":"815 kB"} {"level":"info","ts":"2026-03-30T21:09:48.268527Z","caller":"rafthttp/snapshot_sender.go:82","msg":"sending database snapshot","snapshot-index":512,"remote-peer-id":"7ba82bfa2d9da6c2","bytes":815105,"size":"815 kB"} {"level":"info","ts":"2026-03-30T21:09:48.281421Z","caller":"etcdserver/snapshot_merge.go:64","msg":"sent database snapshot to writer","bytes":806912,"size":"807 kB"} {"level":"info","ts":"2026-03-30T21:09:48.289747Z","caller":"rafthttp/snapshot_sender.go:131","msg":"sent database snapshot","snapshot-index":512,"remote-peer-id":"7ba82bfa2d9da6c2","bytes":815105,"size":"815 kB"} {"level":"warn","ts":"2026-03-30T21:09:48.299569Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:09:48.299538Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2","error":"EOF"} {"level":"info","ts":"2026-03-30T21:09:48.307773Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"89ba38f964b95aa0","to":"7ba82bfa2d9da6c2","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-30T21:09:48.307807Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.307819Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.309973Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"89ba38f964b95aa0","to":"7ba82bfa2d9da6c2","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-30T21:09:48.310021Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.310048Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.318365Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.323454Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"7ba82bfa2d9da6c2"} {"level":"warn","ts":"2026-03-30T21:09:48.344325Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.38:32870","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:09:48.358082Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"89ba38f964b95aa0 switched to configuration voters=(8910420216259651266 9924307372652190368)"} {"level":"info","ts":"2026-03-30T21:09:48.358238Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"54492f71bf65c400","local-member-id":"89ba38f964b95aa0","promoted-member-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:09:48.358272Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"89ba38f964b95aa0","raft-conf-change":"ConfChangeAddNode","raft-conf-change-node-id":"7ba82bfa2d9da6c2"} {"level":"info","ts":"2026-03-30T21:10:03.051241Z","caller":"traceutil/trace.go:172","msg":"trace[643898192] transaction","detail":"{read_only:false; response_revision:525; number_of_response:1; }","duration":"280.625112ms","start":"2026-03-30T21:10:02.770596Z","end":"2026-03-30T21:10:03.051221Z","steps":["trace[643898192] 'process raft request' (duration: 280.464601ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:03.146286Z","caller":"traceutil/trace.go:172","msg":"trace[1348076138] linearizableReadLoop","detail":"{readStateIndex:555; appliedIndex:556; }","duration":"177.245841ms","start":"2026-03-30T21:10:02.969015Z","end":"2026-03-30T21:10:03.146261Z","steps":["trace[1348076138] 'read index received' (duration: 177.239841ms)","trace[1348076138] 'applied index is now lower than readState.Index' (duration: 4.68µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:10:03.146458Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"177.436082ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:10:03.146528Z","caller":"traceutil/trace.go:172","msg":"trace[62373234] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:525; }","duration":"177.517604ms","start":"2026-03-30T21:10:02.968999Z","end":"2026-03-30T21:10:03.146517Z","steps":["trace[62373234] 'agreement among raft nodes before linearized reading' (duration: 177.337411ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:03.146504Z","caller":"traceutil/trace.go:172","msg":"trace[556286571] transaction","detail":"{read_only:false; response_revision:526; number_of_response:1; }","duration":"339.928766ms","start":"2026-03-30T21:10:02.806547Z","end":"2026-03-30T21:10:03.146476Z","steps":["trace[556286571] 'process raft request' (duration: 339.761503ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:10:03.146712Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:10:02.806512Z","time spent":"340.096487ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:10:03.149857Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"117.647414ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-30T21:10:03.149927Z","caller":"traceutil/trace.go:172","msg":"trace[1833272816] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:526; }","duration":"117.732736ms","start":"2026-03-30T21:10:03.032181Z","end":"2026-03-30T21:10:03.149914Z","steps":["trace[1833272816] 'agreement among raft nodes before linearized reading' (duration: 115.711921ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:10.869865Z","caller":"traceutil/trace.go:172","msg":"trace[1343270249] transaction","detail":"{read_only:false; response_revision:571; number_of_response:1; }","duration":"101.277897ms","start":"2026-03-30T21:10:10.768566Z","end":"2026-03-30T21:10:10.869844Z","steps":["trace[1343270249] 'process raft request' (duration: 101.109996ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:10.873875Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"info","ts":"2026-03-30T21:10:13.933086Z","caller":"traceutil/trace.go:172","msg":"trace[733412393] transaction","detail":"{read_only:false; response_revision:588; number_of_response:1; }","duration":"195.766609ms","start":"2026-03-30T21:10:13.737298Z","end":"2026-03-30T21:10:13.933064Z","steps":["trace[733412393] 'process raft request' (duration: 195.611957ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:18.290436Z","caller":"etcdserver/server.go:1856","msg":"sent merged snapshot","from":"89ba38f964b95aa0","to":"7ba82bfa2d9da6c2","bytes":815105,"size":"815 kB","took":"30.022008821s"} {"level":"warn","ts":"2026-03-30T21:10:29.396340Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.157:50524","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:10:30.188499Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"89ba38f964b95aa0 switched to configuration voters=(8910420216259651266 9924307372652190368) learners=(16102761352595172228)"} {"level":"info","ts":"2026-03-30T21:10:30.188692Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"54492f71bf65c400","local-member-id":"89ba38f964b95aa0","added-peer-id":"df7880225452eb84","added-peer-peer-urls":["https://199.204.45.157:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-30T21:10:30.188731Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:30.188754Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:30.188896Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:30.188926Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84","remote-peer-urls":["https://199.204.45.157:2380"]} {"level":"info","ts":"2026-03-30T21:10:30.188952Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"89ba38f964b95aa0","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:30.189136Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:30.189144Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:30.189180Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:30.189203Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"error","ts":"2026-03-30T21:10:30.252464Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*peerMemberPromoteHandler).ServeHTTP\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/peer.go:140\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-30T21:10:30.252536Z","caller":"etcdhttp/peer.go:152","msg":"failed to promote a member","member-id":"df7880225452eb84","error":"etcdserver: can only promote a learner member which is in sync with leader"} {"level":"warn","ts":"2026-03-30T21:10:30.743290Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.157:50648","server-name":"","error":"EOF"} {"level":"error","ts":"2026-03-30T21:10:30.743828Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-30T21:10:31.244057Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-30T21:10:31.744823Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*peerMemberPromoteHandler).ServeHTTP\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/peer.go:140\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-30T21:10:31.744912Z","caller":"etcdhttp/peer.go:152","msg":"failed to promote a member","member-id":"df7880225452eb84","error":"etcdserver: can only promote a learner member which is in sync with leader"} {"level":"warn","ts":"2026-03-30T21:10:32.100552Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://199.204.45.157:2380/version","remote-member-id":"df7880225452eb84","error":"Get \"https://199.204.45.157:2380/version\": dial tcp 199.204.45.157:2380: connect: connection refused"} {"level":"warn","ts":"2026-03-30T21:10:32.100631Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"df7880225452eb84","error":"Get \"https://199.204.45.157:2380/version\": dial tcp 199.204.45.157:2380: connect: connection refused"} {"level":"error","ts":"2026-03-30T21:10:32.241732Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-30T21:10:32.742210Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"info","ts":"2026-03-30T21:10:32.948420Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:32.948480Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:32.949722Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:32.956055Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"89ba38f964b95aa0","to":"df7880225452eb84","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-30T21:10:32.956147Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:32.958631Z","caller":"etcdserver/server.go:1838","msg":"sending merged snapshot","from":"89ba38f964b95aa0","to":"df7880225452eb84","bytes":1045032,"size":"1.0 MB"} {"level":"info","ts":"2026-03-30T21:10:32.958844Z","caller":"rafthttp/snapshot_sender.go:82","msg":"sending database snapshot","snapshot-index":680,"remote-peer-id":"df7880225452eb84","bytes":1045032,"size":"1.0 MB"} {"level":"info","ts":"2026-03-30T21:10:32.960200Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"89ba38f964b95aa0","to":"df7880225452eb84","stream-type":"stream Message"} {"level":"info","ts":"2026-03-30T21:10:32.960251Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:32.964079Z","caller":"etcdserver/snapshot_merge.go:64","msg":"sent database snapshot to writer","bytes":1036288,"size":"1.0 MB"} {"level":"info","ts":"2026-03-30T21:10:32.973101Z","caller":"rafthttp/snapshot_sender.go:131","msg":"sent database snapshot","snapshot-index":680,"remote-peer-id":"df7880225452eb84","bytes":1045032,"size":"1.0 MB"} {"level":"warn","ts":"2026-03-30T21:10:32.993788Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:10:32.994012Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84","error":"EOF"} {"level":"info","ts":"2026-03-30T21:10:33.007471Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:33.014152Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"89ba38f964b95aa0","to":"df7880225452eb84","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-30T21:10:33.014183Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:33.014195Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:33.014792Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:33.018383Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"89ba38f964b95aa0","to":"df7880225452eb84","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-30T21:10:33.018406Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:33.018419Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"89ba38f964b95aa0","remote-peer-id":"df7880225452eb84"} {"level":"warn","ts":"2026-03-30T21:10:33.242689Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.157:46900","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:10:33.245160Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"89ba38f964b95aa0 switched to configuration voters=(8910420216259651266 9924307372652190368 16102761352595172228)"} {"level":"info","ts":"2026-03-30T21:10:33.245370Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"54492f71bf65c400","local-member-id":"89ba38f964b95aa0","promoted-member-id":"df7880225452eb84"} {"level":"info","ts":"2026-03-30T21:10:33.245452Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"89ba38f964b95aa0","raft-conf-change":"ConfChangeAddNode","raft-conf-change-node-id":"df7880225452eb84"} {"level":"warn","ts":"2026-03-30T21:10:36.002442Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"118.191286ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:10:36.002564Z","caller":"traceutil/trace.go:172","msg":"trace[1890400974] range","detail":"{range_begin:/registry/controllers; range_end:; response_count:0; response_revision:646; }","duration":"118.736933ms","start":"2026-03-30T21:10:35.883810Z","end":"2026-03-30T21:10:36.002547Z","steps":["trace[1890400974] 'range keys from in-memory index tree' (duration: 117.373517ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:37.594349Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"warn","ts":"2026-03-30T21:10:43.666359Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"172.849009ms"} {"level":"warn","ts":"2026-03-30T21:10:43.666434Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"172.929019ms"} {"level":"warn","ts":"2026-03-30T21:10:43.674154Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"380.35768ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T21:10:43.674523Z","caller":"traceutil/trace.go:172","msg":"trace[990685106] linearizableReadLoop","detail":"{readStateIndex:752; appliedIndex:751; }","duration":"471.956394ms","start":"2026-03-30T21:10:43.202549Z","end":"2026-03-30T21:10:43.674506Z","steps":["trace[990685106] 'read index received' (duration: 451.54021ms)","trace[990685106] 'applied index is now lower than readState.Index' (duration: 20.413714ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:10:43.674636Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"472.075505ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:10:43.674663Z","caller":"traceutil/trace.go:172","msg":"trace[1866270584] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:697; }","duration":"472.113026ms","start":"2026-03-30T21:10:43.202543Z","end":"2026-03-30T21:10:43.674656Z","steps":["trace[1866270584] 'agreement among raft nodes before linearized reading' (duration: 472.056425ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:43.675326Z","caller":"traceutil/trace.go:172","msg":"trace[1026842612] transaction","detail":"{read_only:false; response_revision:698; number_of_response:1; }","duration":"385.202467ms","start":"2026-03-30T21:10:43.290112Z","end":"2026-03-30T21:10:43.675315Z","steps":["trace[1026842612] 'process raft request' (duration: 384.515639ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:43.675325Z","caller":"traceutil/trace.go:172","msg":"trace[370082163] transaction","detail":"{read_only:false; response_revision:699; number_of_response:1; }","duration":"304.725825ms","start":"2026-03-30T21:10:43.370586Z","end":"2026-03-30T21:10:43.675311Z","steps":["trace[370082163] 'process raft request' (duration: 304.453731ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:10:43.675622Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:10:43.370532Z","time spent":"305.044508ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:10:43.675759Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:10:43.290084Z","time spent":"385.301788ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:10:43.675987Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"249.078659ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:10:43.676170Z","caller":"traceutil/trace.go:172","msg":"trace[114926897] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:699; }","duration":"249.19787ms","start":"2026-03-30T21:10:43.426881Z","end":"2026-03-30T21:10:43.676079Z","steps":["trace[114926897] 'agreement among raft nodes before linearized reading' (duration: 248.870506ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:10:43.676118Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"210.113163ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:10:43.676283Z","caller":"traceutil/trace.go:172","msg":"trace[143374850] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:699; }","duration":"210.279716ms","start":"2026-03-30T21:10:43.465993Z","end":"2026-03-30T21:10:43.676272Z","steps":["trace[143374850] 'agreement among raft nodes before linearized reading' (duration: 210.012972ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:10:52.981735Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"info","ts":"2026-03-30T21:11:02.973702Z","caller":"etcdserver/server.go:1856","msg":"sent merged snapshot","from":"89ba38f964b95aa0","to":"df7880225452eb84","bytes":1045032,"size":"1.0 MB","took":"30.015063863s"} {"level":"warn","ts":"2026-03-30T21:11:11.373825Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"110.280815ms"} {"level":"warn","ts":"2026-03-30T21:11:11.373951Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"110.414636ms"} {"level":"info","ts":"2026-03-30T21:11:11.436521Z","caller":"traceutil/trace.go:172","msg":"trace[82297163] transaction","detail":"{read_only:false; response_revision:868; number_of_response:1; }","duration":"341.491986ms","start":"2026-03-30T21:11:11.095009Z","end":"2026-03-30T21:11:11.436501Z","steps":["trace[82297163] 'process raft request' (duration: 341.381505ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.436678Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.094984Z","time spent":"341.634248ms","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":6315,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:11.439000Z","caller":"traceutil/trace.go:172","msg":"trace[1261794656] transaction","detail":"{read_only:false; response_revision:869; number_of_response:1; }","duration":"335.171111ms","start":"2026-03-30T21:11:11.103809Z","end":"2026-03-30T21:11:11.438980Z","steps":["trace[1261794656] 'process raft request' (duration: 335.027229ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.439176Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.103797Z","time spent":"335.303892ms","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":9490,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:11.452428Z","caller":"traceutil/trace.go:172","msg":"trace[1143740795] linearizableReadLoop","detail":"{readStateIndex:939; appliedIndex:941; }","duration":"341.668939ms","start":"2026-03-30T21:11:11.110745Z","end":"2026-03-30T21:11:11.452414Z","steps":["trace[1143740795] 'read index received' (duration: 341.665079ms)","trace[1143740795] 'applied index is now lower than readState.Index' (duration: 3.48µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:11.452556Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"341.80276ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:11.452577Z","caller":"traceutil/trace.go:172","msg":"trace[1290024118] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:869; }","duration":"341.836811ms","start":"2026-03-30T21:11:11.110736Z","end":"2026-03-30T21:11:11.452572Z","steps":["trace[1290024118] 'agreement among raft nodes before linearized reading' (duration: 341.77496ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.452595Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.110715Z","time spent":"341.877181ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:11:11.452827Z","caller":"traceutil/trace.go:172","msg":"trace[95005830] transaction","detail":"{read_only:false; response_revision:871; number_of_response:1; }","duration":"332.706743ms","start":"2026-03-30T21:11:11.120104Z","end":"2026-03-30T21:11:11.452811Z","steps":["trace[95005830] 'process raft request' (duration: 332.46601ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.452955Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.120077Z","time spent":"332.821774ms","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":28462,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:11.453051Z","caller":"traceutil/trace.go:172","msg":"trace[1577694513] transaction","detail":"{read_only:false; response_revision:870; number_of_response:1; }","duration":"344.681565ms","start":"2026-03-30T21:11:11.108362Z","end":"2026-03-30T21:11:11.453043Z","steps":["trace[1577694513] 'process raft request' (duration: 344.108188ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.453079Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.108351Z","time spent":"344.714875ms","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":8162,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:11.453131Z","caller":"traceutil/trace.go:172","msg":"trace[420773608] transaction","detail":"{read_only:false; response_revision:872; number_of_response:1; }","duration":"331.396007ms","start":"2026-03-30T21:11:11.121730Z","end":"2026-03-30T21:11:11.453126Z","steps":["trace[420773608] 'process raft request' (duration: 330.8861ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.453159Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.121701Z","time spent":"331.447487ms","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":29801,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:11.453130Z","caller":"traceutil/trace.go:172","msg":"trace[1732781812] transaction","detail":"{read_only:false; response_revision:874; number_of_response:1; }","duration":"327.480689ms","start":"2026-03-30T21:11:11.125621Z","end":"2026-03-30T21:11:11.453101Z","steps":["trace[1732781812] 'process raft request' (duration: 327.145695ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.453261Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.125614Z","time spent":"327.58944ms","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":29820,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:11.453219Z","caller":"traceutil/trace.go:172","msg":"trace[2055930665] transaction","detail":"{read_only:false; response_revision:875; number_of_response:1; }","duration":"322.838715ms","start":"2026-03-30T21:11:11.130374Z","end":"2026-03-30T21:11:11.453212Z","steps":["trace[2055930665] 'process raft request' (duration: 322.47085ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.453312Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.130278Z","time spent":"323.021276ms","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":28166,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:11.453381Z","caller":"traceutil/trace.go:172","msg":"trace[424503316] transaction","detail":"{read_only:false; response_revision:873; number_of_response:1; }","duration":"327.454369ms","start":"2026-03-30T21:11:11.125918Z","end":"2026-03-30T21:11:11.453373Z","steps":["trace[424503316] 'process raft request' (duration: 326.811072ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:11.453415Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.125912Z","time spent":"327.48607ms","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":22019,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:11.453476Z","caller":"traceutil/trace.go:172","msg":"trace[2092383111] transaction","detail":"{read_only:false; response_revision:877; number_of_response:1; }","duration":"291.091247ms","start":"2026-03-30T21:11:11.162377Z","end":"2026-03-30T21:11:11.453468Z","steps":["trace[2092383111] 'process raft request' (duration: 290.57184ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:11.453928Z","caller":"traceutil/trace.go:172","msg":"trace[1134080110] transaction","detail":"{read_only:false; response_revision:876; number_of_response:1; }","duration":"298.562925ms","start":"2026-03-30T21:11:11.154798Z","end":"2026-03-30T21:11:11.453361Z","steps":["trace[1134080110] 'process raft request' (duration: 298.0966ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:11.474567Z","caller":"traceutil/trace.go:172","msg":"trace[50868747] transaction","detail":"{read_only:false; response_revision:878; number_of_response:1; }","duration":"291.40767ms","start":"2026-03-30T21:11:11.183145Z","end":"2026-03-30T21:11:11.474553Z","steps":["trace[50868747] 'process raft request' (duration: 291.030235ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:11.502665Z","caller":"traceutil/trace.go:172","msg":"trace[1852215051] transaction","detail":"{read_only:false; response_revision:879; number_of_response:1; }","duration":"218.017826ms","start":"2026-03-30T21:11:11.284616Z","end":"2026-03-30T21:11:11.502634Z","steps":["trace[1852215051] 'process raft request' (duration: 190.138365ms)","trace[1852215051] 'compare' (duration: 27.583248ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:11.503222Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"298.333563ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:11.503269Z","caller":"traceutil/trace.go:172","msg":"trace[2066301496] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:884; }","duration":"298.391933ms","start":"2026-03-30T21:11:11.204866Z","end":"2026-03-30T21:11:11.503258Z","steps":["trace[2066301496] 'agreement among raft nodes before linearized reading' (duration: 298.312472ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:12.445562Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983281,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:12.947045Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983281,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:13.448201Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983281,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:13.949218Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983281,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:14.152669Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000698322s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-30T21:11:14.152827Z","caller":"traceutil/trace.go:172","msg":"trace[1373219926] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000787083s","start":"2026-03-30T21:11:12.151938Z","end":"2026-03-30T21:11:14.152726Z","steps":["trace[1373219926] 'agreement among raft nodes before linearized reading' (duration: 2.000695712s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.152935Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:12.151901Z","time spent":"2.001022627s","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 21:11:14 WARNING: [core] [Server #4]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-30T21:11:14.449888Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983281,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:14.522438Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.828182705s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:5"} {"level":"warn","ts":"2026-03-30T21:11:14.522533Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.319438417s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:14.522575Z","caller":"traceutil/trace.go:172","msg":"trace[1377309109] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:890; }","duration":"2.319482187s","start":"2026-03-30T21:11:12.203080Z","end":"2026-03-30T21:11:14.522562Z","steps":["trace[1377309109] 'range keys from in-memory index tree' (duration: 2.319373697s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.523574Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.818853176s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:14.524009Z","caller":"traceutil/trace.go:172","msg":"trace[486877331] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:890; }","duration":"1.820396536s","start":"2026-03-30T21:11:12.703590Z","end":"2026-03-30T21:11:14.523987Z","steps":["trace[486877331] 'range keys from in-memory index tree' (duration: 1.818803736s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.563778Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"2.868036989s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T21:11:14.563922Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"2.670061373s"} {"level":"warn","ts":"2026-03-30T21:11:14.563956Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"2.670100154s"} {"level":"info","ts":"2026-03-30T21:11:14.564597Z","caller":"traceutil/trace.go:172","msg":"trace[640259849] transaction","detail":"{read_only:false; response_revision:891; number_of_response:1; }","duration":"3.000289644s","start":"2026-03-30T21:11:11.564272Z","end":"2026-03-30T21:11:14.564562Z","steps":["trace[640259849] 'process raft request' (duration: 2.999942241s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.564764Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.564251Z","time spent":"3.000414216s","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":9125,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:14.564764Z","caller":"traceutil/trace.go:172","msg":"trace[1414643107] transaction","detail":"{read_only:false; response_revision:892; number_of_response:1; }","duration":"2.999541906s","start":"2026-03-30T21:11:11.564951Z","end":"2026-03-30T21:11:14.564493Z","steps":["trace[1414643107] 'process raft request' (duration: 2.999428505s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.565057Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.564933Z","time spent":"2.99993351s","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":58844,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:14.584943Z","caller":"traceutil/trace.go:172","msg":"trace[1277403425] transaction","detail":"{read_only:false; response_revision:893; number_of_response:1; }","duration":"3.019425253s","start":"2026-03-30T21:11:11.565446Z","end":"2026-03-30T21:11:14.584871Z","steps":["trace[1277403425] 'process raft request' (duration: 3.0192395s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:14.585004Z","caller":"traceutil/trace.go:172","msg":"trace[10749408] transaction","detail":"{read_only:false; response_revision:894; number_of_response:1; }","duration":"3.010344394s","start":"2026-03-30T21:11:11.574647Z","end":"2026-03-30T21:11:14.584991Z","steps":["trace[10749408] 'process raft request' (duration: 3.010153192s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.585131Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.565437Z","time spent":"3.019575404s","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":79947,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:14.585195Z","caller":"traceutil/trace.go:172","msg":"trace[74990091] transaction","detail":"{read_only:false; response_revision:895; number_of_response:1; }","duration":"2.985672661s","start":"2026-03-30T21:11:11.599514Z","end":"2026-03-30T21:11:14.585186Z","steps":["trace[74990091] 'process raft request' (duration: 2.985471498s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.585248Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.599461Z","time spent":"2.985767151s","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":150480,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:14.585290Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.574634Z","time spent":"3.010443806s","remote":"127.0.0.1:45496","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":17062,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:14.592073Z","caller":"traceutil/trace.go:172","msg":"trace[765374009] linearizableReadLoop","detail":"{readStateIndex:993; appliedIndex:1007; }","duration":"2.647414283s","start":"2026-03-30T21:11:11.944636Z","end":"2026-03-30T21:11:14.592050Z","steps":["trace[765374009] 'read index received' (duration: 2.647408683s)","trace[765374009] 'applied index is now lower than readState.Index' (duration: 4.97µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:14.592253Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.647619124s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:11:14.592297Z","caller":"traceutil/trace.go:172","msg":"trace[1096049847] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:895; }","duration":"2.647664515s","start":"2026-03-30T21:11:11.944622Z","end":"2026-03-30T21:11:14.592286Z","steps":["trace[1096049847] 'agreement among raft nodes before linearized reading' (duration: 2.647498673s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.592325Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.944600Z","time spent":"2.647717086s","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T21:11:14.592530Z","caller":"traceutil/trace.go:172","msg":"trace[1753366946] transaction","detail":"{read_only:false; response_revision:896; number_of_response:1; }","duration":"2.697750421s","start":"2026-03-30T21:11:11.894768Z","end":"2026-03-30T21:11:14.592518Z","steps":["trace[1753366946] 'process raft request' (duration: 2.697367806s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.592592Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.894742Z","time spent":"2.697819503s","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:14.592670Z","caller":"traceutil/trace.go:172","msg":"trace[1920213204] transaction","detail":"{read_only:false; response_revision:897; number_of_response:1; }","duration":"2.689932828s","start":"2026-03-30T21:11:11.902726Z","end":"2026-03-30T21:11:14.592659Z","steps":["trace[1920213204] 'process raft request' (duration: 2.689635745s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:14.592744Z","caller":"traceutil/trace.go:172","msg":"trace[278536105] transaction","detail":"{read_only:false; response_revision:898; number_of_response:1; }","duration":"2.420356099s","start":"2026-03-30T21:11:12.172365Z","end":"2026-03-30T21:11:14.592721Z","steps":["trace[278536105] 'process raft request' (duration: 2.420150267s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:14.592730Z","caller":"traceutil/trace.go:172","msg":"trace[1627687365] transaction","detail":"{read_only:false; response_revision:899; number_of_response:1; }","duration":"436.934051ms","start":"2026-03-30T21:11:14.155785Z","end":"2026-03-30T21:11:14.592719Z","steps":["trace[1627687365] 'process raft request' (duration: 436.84764ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.592772Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.902717Z","time spent":"2.68999002s","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:14.592880Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:12.172332Z","time spent":"2.420483231s","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":676,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:14.592921Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:14.155762Z","time spent":"437.132094ms","remote":"127.0.0.1:45538","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":778,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:11:14.607185Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.661739993s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:11:14.607252Z","caller":"traceutil/trace.go:172","msg":"trace[1176753177] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:899; }","duration":"2.661815924s","start":"2026-03-30T21:11:11.945423Z","end":"2026-03-30T21:11:14.607239Z","steps":["trace[1176753177] 'agreement among raft nodes before linearized reading' (duration: 2.661633592s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.607288Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:11.945399Z","time spent":"2.661879995s","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:14.607340Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.326900939s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:11:14.607818Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"259.52283ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/daemonsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:14.607828Z","caller":"traceutil/trace.go:172","msg":"trace[453379106] range","detail":"{range_begin:/registry/resourceslices; range_end:; response_count:0; response_revision:902; }","duration":"1.327381714s","start":"2026-03-30T21:11:13.280429Z","end":"2026-03-30T21:11:14.607810Z","steps":["trace[453379106] 'agreement among raft nodes before linearized reading' (duration: 1.326788276s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:14.607860Z","caller":"traceutil/trace.go:172","msg":"trace[996055183] range","detail":"{range_begin:/registry/daemonsets; range_end:; response_count:0; response_revision:902; }","duration":"259.56102ms","start":"2026-03-30T21:11:14.348286Z","end":"2026-03-30T21:11:14.607847Z","steps":["trace[996055183] 'agreement among raft nodes before linearized reading' (duration: 259.48337ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.607895Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:13.280406Z","time spent":"1.327456554s","remote":"127.0.0.1:46378","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/resourceslices\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:14.607950Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"319.029217ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:14.607998Z","caller":"traceutil/trace.go:172","msg":"trace[1159191447] range","detail":"{range_begin:/registry/resourceclaims; range_end:; response_count:0; response_revision:902; }","duration":"319.068798ms","start":"2026-03-30T21:11:14.288913Z","end":"2026-03-30T21:11:14.607981Z","steps":["trace[1159191447] 'agreement among raft nodes before linearized reading' (duration: 318.967287ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.608037Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:14.288896Z","time spent":"319.134119ms","remote":"127.0.0.1:46356","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/resourceclaims\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:14.607339Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.501144223s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:14.608393Z","caller":"traceutil/trace.go:172","msg":"trace[828352209] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:902; }","duration":"1.502213475s","start":"2026-03-30T21:11:13.106166Z","end":"2026-03-30T21:11:14.608379Z","steps":["trace[828352209] 'agreement among raft nodes before linearized reading' (duration: 1.501121963s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.608903Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.488714743s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:14.608946Z","caller":"traceutil/trace.go:172","msg":"trace[508699618] range","detail":"{range_begin:/registry/validatingadmissionpolicybindings; range_end:; response_count:0; response_revision:902; }","duration":"2.488757883s","start":"2026-03-30T21:11:12.120179Z","end":"2026-03-30T21:11:14.608937Z","steps":["trace[508699618] 'agreement among raft nodes before linearized reading' (duration: 2.488695753s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:14.608967Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:12.120162Z","time spent":"2.488798514s","remote":"127.0.0.1:46318","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":28,"request content":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:14.609318Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:13.106142Z","time spent":"1.502426278s","remote":"127.0.0.1:45428","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:11:16.666695Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38902","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:16.744953Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38930","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:16.834610Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38950","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:16.845312Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38972","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:16.859739Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:38984","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:16.884891Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39010","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:16.987975Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39022","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.061704Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39040","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.081881Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39056","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.126433Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39064","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.149019Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39086","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.169806Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39088","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.229261Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39116","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.252811Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39138","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.302953Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39150","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.428614Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39180","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.448433Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39192","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.471694Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39218","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.496465Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39246","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.519815Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39258","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:17.537460Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:39268","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:19.016012Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"336.058511ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:11:19.016076Z","caller":"traceutil/trace.go:172","msg":"trace[116002914] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:967; }","duration":"336.127821ms","start":"2026-03-30T21:11:18.679934Z","end":"2026-03-30T21:11:19.016062Z","steps":["trace[116002914] 'agreement among raft nodes before linearized reading' (duration: 30.99547ms)","trace[116002914] 'range keys from in-memory index tree' (duration: 304.99067ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:19.016129Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:18.679859Z","time spent":"336.262972ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:19.016607Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"306.067913ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:11:19.016733Z","caller":"traceutil/trace.go:172","msg":"trace[63555264] transaction","detail":"{read_only:false; response_revision:968; number_of_response:1; }","duration":"337.639919ms","start":"2026-03-30T21:11:18.679073Z","end":"2026-03-30T21:11:19.016713Z","steps":["trace[63555264] 'process raft request' (duration: 31.354703ms)","trace[63555264] 'compare' (duration: 305.458977ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:19.016846Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:18.679056Z","time spent":"337.74662ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":426,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:19.383474Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983473,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:19.384144Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"424.015466ms"} {"level":"warn","ts":"2026-03-30T21:11:19.384214Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"424.093878ms"} {"level":"info","ts":"2026-03-30T21:11:19.386140Z","caller":"traceutil/trace.go:172","msg":"trace[1225082614] transaction","detail":"{read_only:false; response_revision:969; number_of_response:1; }","duration":"590.943464ms","start":"2026-03-30T21:11:18.794112Z","end":"2026-03-30T21:11:19.385056Z","steps":["trace[1225082614] 'process raft request' (duration: 590.354216ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:19.386289Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:18.794073Z","time spent":"592.131057ms","remote":"127.0.0.1:45568","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":1036161,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:19.795650Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"211.411576ms"} {"level":"warn","ts":"2026-03-30T21:11:19.795709Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"211.479747ms"} {"level":"info","ts":"2026-03-30T21:11:19.796042Z","caller":"traceutil/trace.go:172","msg":"trace[1389471408] linearizableReadLoop","detail":"{readStateIndex:1130; appliedIndex:1131; }","duration":"913.551682ms","start":"2026-03-30T21:11:18.882462Z","end":"2026-03-30T21:11:19.796013Z","steps":["trace[1389471408] 'read index received' (duration: 913.542222ms)","trace[1389471408] 'applied index is now lower than readState.Index' (duration: 8.39µs)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:11:19.796190Z","caller":"traceutil/trace.go:172","msg":"trace[367175648] transaction","detail":"{read_only:false; response_revision:970; number_of_response:1; }","duration":"776.362459ms","start":"2026-03-30T21:11:19.019809Z","end":"2026-03-30T21:11:19.796171Z","steps":["trace[367175648] 'process raft request' (duration: 776.175316ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:19.796312Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:19.019793Z","time spent":"776.46334ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:19.796816Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"914.037367ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/servicecidrs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:19.796851Z","caller":"traceutil/trace.go:172","msg":"trace[1795211285] range","detail":"{range_begin:/registry/servicecidrs; range_end:; response_count:0; response_revision:971; }","duration":"914.388121ms","start":"2026-03-30T21:11:18.882455Z","end":"2026-03-30T21:11:19.796843Z","steps":["trace[1795211285] 'agreement among raft nodes before linearized reading' (duration: 913.997197ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:19.796878Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:18.882436Z","time spent":"914.436173ms","remote":"127.0.0.1:45924","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/servicecidrs\" limit:1 "} {"level":"info","ts":"2026-03-30T21:11:19.813216Z","caller":"traceutil/trace.go:172","msg":"trace[1584963864] transaction","detail":"{read_only:false; response_revision:972; number_of_response:1; }","duration":"371.41378ms","start":"2026-03-30T21:11:19.441785Z","end":"2026-03-30T21:11:19.813199Z","steps":["trace[1584963864] 'process raft request' (duration: 371.328639ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:19.813336Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:19.441763Z","time spent":"371.520892ms","remote":"127.0.0.1:45966","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":897,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:11:19.814056Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"708.176798ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:19.814150Z","caller":"traceutil/trace.go:172","msg":"trace[1264853414] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:972; }","duration":"708.276829ms","start":"2026-03-30T21:11:19.105855Z","end":"2026-03-30T21:11:19.814132Z","steps":["trace[1264853414] 'agreement among raft nodes before linearized reading' (duration: 708.137298ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:19.814272Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:19.105840Z","time spent":"708.41769ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:11:19.814291Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"611.140973ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:19.814471Z","caller":"traceutil/trace.go:172","msg":"trace[648191423] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:972; }","duration":"611.348556ms","start":"2026-03-30T21:11:19.203021Z","end":"2026-03-30T21:11:19.814369Z","steps":["trace[648191423] 'agreement among raft nodes before linearized reading' (duration: 611.061913ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:21.332182Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"117.454898ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-30T21:11:21.332278Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"129.592231ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:21.332359Z","caller":"traceutil/trace.go:172","msg":"trace[1222997100] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:984; }","duration":"129.706434ms","start":"2026-03-30T21:11:21.202635Z","end":"2026-03-30T21:11:21.332342Z","steps":["trace[1222997100] 'range keys from in-memory index tree' (duration: 116.605776ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:21.352028Z","caller":"traceutil/trace.go:172","msg":"trace[554269130] transaction","detail":"{read_only:false; response_revision:985; number_of_response:1; }","duration":"128.196496ms","start":"2026-03-30T21:11:21.222704Z","end":"2026-03-30T21:11:21.350900Z","steps":["trace[554269130] 'process raft request' (duration: 127.65056ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:21.909415Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"108.496002ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:11:21.909489Z","caller":"traceutil/trace.go:172","msg":"trace[41753619] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:986; }","duration":"108.578523ms","start":"2026-03-30T21:11:21.800896Z","end":"2026-03-30T21:11:21.909475Z","steps":["trace[41753619] 'range keys from in-memory index tree' (duration: 107.626731ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:21.964830Z","caller":"traceutil/trace.go:172","msg":"trace[1006238741] transaction","detail":"{read_only:false; response_revision:987; number_of_response:1; }","duration":"118.444599ms","start":"2026-03-30T21:11:21.846365Z","end":"2026-03-30T21:11:21.964809Z","steps":["trace[1006238741] 'process raft request' (duration: 117.816092ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:22.616195Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"510.831009ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:22.616275Z","caller":"traceutil/trace.go:172","msg":"trace[593150905] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:988; }","duration":"510.9245ms","start":"2026-03-30T21:11:22.105336Z","end":"2026-03-30T21:11:22.616261Z","steps":["trace[593150905] 'agreement among raft nodes before linearized reading' (duration: 69.510258ms)","trace[593150905] 'range keys from in-memory index tree' (duration: 441.27745ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:22.616306Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:22.105318Z","time spent":"510.98184ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:11:22.616362Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"413.099685ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:22.616540Z","caller":"traceutil/trace.go:172","msg":"trace[820730421] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:988; }","duration":"413.202976ms","start":"2026-03-30T21:11:22.203227Z","end":"2026-03-30T21:11:22.616430Z","steps":["trace[820730421] 'range keys from in-memory index tree' (duration: 413.000584ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:22.618235Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"443.933602ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:11:22.618488Z","caller":"traceutil/trace.go:172","msg":"trace[1222112652] transaction","detail":"{read_only:false; response_revision:989; number_of_response:1; }","duration":"591.941384ms","start":"2026-03-30T21:11:22.026484Z","end":"2026-03-30T21:11:22.618425Z","steps":["trace[1222112652] 'process raft request' (duration: 147.670707ms)","trace[1222112652] 'compare' (duration: 442.992091ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:22.618747Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:22.026462Z","time spent":"592.189566ms","remote":"127.0.0.1:46274","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":1627,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:11:22.880257Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"162.986009ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:22.880345Z","caller":"traceutil/trace.go:172","msg":"trace[295620116] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions; range_end:; response_count:0; response_revision:990; }","duration":"163.082381ms","start":"2026-03-30T21:11:22.717243Z","end":"2026-03-30T21:11:22.880325Z","steps":["trace[295620116] 'agreement among raft nodes before linearized reading' (duration: 73.355942ms)","trace[295620116] 'range keys from in-memory index tree' (duration: 89.603096ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:11:22.880537Z","caller":"traceutil/trace.go:172","msg":"trace[860225300] transaction","detail":"{read_only:false; response_revision:991; number_of_response:1; }","duration":"248.505596ms","start":"2026-03-30T21:11:22.632008Z","end":"2026-03-30T21:11:22.880514Z","steps":["trace[860225300] 'process raft request' (duration: 158.730109ms)","trace[860225300] 'compare' (duration: 89.650836ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:23.282107Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"99.080318ms"} {"level":"warn","ts":"2026-03-30T21:11:23.282169Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"99.149909ms"} {"level":"info","ts":"2026-03-30T21:11:23.283478Z","caller":"traceutil/trace.go:172","msg":"trace[1230330298] transaction","detail":"{read_only:false; response_revision:994; number_of_response:1; }","duration":"283.23539ms","start":"2026-03-30T21:11:23.000224Z","end":"2026-03-30T21:11:23.283460Z","steps":["trace[1230330298] 'process raft request' (duration: 282.241218ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:23.285509Z","caller":"traceutil/trace.go:172","msg":"trace[1066966391] linearizableReadLoop","detail":"{readStateIndex:1157; appliedIndex:1158; }","duration":"187.376399ms","start":"2026-03-30T21:11:23.098110Z","end":"2026-03-30T21:11:23.285486Z","steps":["trace[1066966391] 'read index received' (duration: 187.369389ms)","trace[1066966391] 'applied index is now lower than readState.Index' (duration: 5.69µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:23.285973Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"187.836454ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-30T21:11:23.286075Z","caller":"traceutil/trace.go:172","msg":"trace[1003333261] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:994; }","duration":"187.956715ms","start":"2026-03-30T21:11:23.098106Z","end":"2026-03-30T21:11:23.286062Z","steps":["trace[1003333261] 'agreement among raft nodes before linearized reading' (duration: 187.642091ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:23.755681Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"99.586224ms"} {"level":"warn","ts":"2026-03-30T21:11:23.755759Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"99.678185ms"} {"level":"info","ts":"2026-03-30T21:11:23.757790Z","caller":"traceutil/trace.go:172","msg":"trace[1018135895] transaction","detail":"{read_only:false; response_revision:997; number_of_response:1; }","duration":"298.681754ms","start":"2026-03-30T21:11:23.458935Z","end":"2026-03-30T21:11:23.757617Z","steps":["trace[1018135895] 'process raft request' (duration: 296.873212ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:24.115554Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"155.799273ms"} {"level":"warn","ts":"2026-03-30T21:11:24.115628Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"155.900964ms"} {"level":"info","ts":"2026-03-30T21:11:24.116180Z","caller":"traceutil/trace.go:172","msg":"trace[2046072845] transaction","detail":"{read_only:false; response_revision:999; number_of_response:1; }","duration":"350.930396ms","start":"2026-03-30T21:11:23.765227Z","end":"2026-03-30T21:11:24.116157Z","steps":["trace[2046072845] 'process raft request' (duration: 350.770104ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:24.116224Z","caller":"traceutil/trace.go:172","msg":"trace[759802361] transaction","detail":"{read_only:false; response_revision:1000; number_of_response:1; }","duration":"349.823162ms","start":"2026-03-30T21:11:23.766368Z","end":"2026-03-30T21:11:24.116191Z","steps":["trace[759802361] 'process raft request' (duration: 349.66332ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:24.116358Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:23.765213Z","time spent":"351.048347ms","remote":"127.0.0.1:45538","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":775,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:24.116185Z","caller":"traceutil/trace.go:172","msg":"trace[657917084] transaction","detail":"{read_only:false; response_revision:1001; number_of_response:1; }","duration":"348.389345ms","start":"2026-03-30T21:11:23.767775Z","end":"2026-03-30T21:11:24.116164Z","steps":["trace[657917084] 'process raft request' (duration: 348.343374ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:24.116343Z","caller":"traceutil/trace.go:172","msg":"trace[313221611] transaction","detail":"{read_only:false; response_revision:998; number_of_response:1; }","duration":"351.813916ms","start":"2026-03-30T21:11:23.764510Z","end":"2026-03-30T21:11:24.116324Z","steps":["trace[313221611] 'process raft request' (duration: 351.377271ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:24.116427Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:23.766355Z","time spent":"349.988374ms","remote":"127.0.0.1:45822","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":2879,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:24.117015Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:23.764485Z","time spent":"352.21831ms","remote":"127.0.0.1:45538","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":933,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:11:24.116810Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:23.767756Z","time spent":"348.85596ms","remote":"127.0.0.1:45740","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3268,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:24.205956Z","caller":"traceutil/trace.go:172","msg":"trace[1002669338] linearizableReadLoop","detail":"{readStateIndex:1163; appliedIndex:1166; }","duration":"200.318513ms","start":"2026-03-30T21:11:24.005617Z","end":"2026-03-30T21:11:24.205935Z","steps":["trace[1002669338] 'read index received' (duration: 200.313353ms)","trace[1002669338] 'applied index is now lower than readState.Index' (duration: 4.18µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:24.207232Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"201.605099ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:11:24.207291Z","caller":"traceutil/trace.go:172","msg":"trace[1220015878] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1001; }","duration":"201.679449ms","start":"2026-03-30T21:11:24.005601Z","end":"2026-03-30T21:11:24.207280Z","steps":["trace[1220015878] 'agreement among raft nodes before linearized reading' (duration: 200.435785ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:24.249546Z","caller":"traceutil/trace.go:172","msg":"trace[1980091533] transaction","detail":"{read_only:false; response_revision:1002; number_of_response:1; }","duration":"290.911471ms","start":"2026-03-30T21:11:23.958621Z","end":"2026-03-30T21:11:24.249532Z","steps":["trace[1980091533] 'process raft request' (duration: 290.81265ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:24.504234Z","caller":"traceutil/trace.go:172","msg":"trace[1871312868] linearizableReadLoop","detail":"{readStateIndex:1166; appliedIndex:1167; }","duration":"298.191378ms","start":"2026-03-30T21:11:24.206012Z","end":"2026-03-30T21:11:24.504203Z","steps":["trace[1871312868] 'read index received' (duration: 298.184257ms)","trace[1871312868] 'applied index is now lower than readState.Index' (duration: 5.801µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:24.504294Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"301.366665ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:24.504372Z","caller":"traceutil/trace.go:172","msg":"trace[1686177421] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1002; }","duration":"301.454176ms","start":"2026-03-30T21:11:24.202898Z","end":"2026-03-30T21:11:24.504353Z","steps":["trace[1686177421] 'agreement among raft nodes before linearized reading' (duration: 301.327405ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:24.504381Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"399.614094ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:24.504425Z","caller":"traceutil/trace.go:172","msg":"trace[1955071945] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1002; }","duration":"399.670094ms","start":"2026-03-30T21:11:24.104741Z","end":"2026-03-30T21:11:24.504411Z","steps":["trace[1955071945] 'agreement among raft nodes before linearized reading' (duration: 399.571793ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:24.504472Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:24.104727Z","time spent":"399.734085ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:11:24.505043Z","caller":"traceutil/trace.go:172","msg":"trace[1450384589] transaction","detail":"{read_only:false; response_revision:1003; number_of_response:1; }","duration":"294.103339ms","start":"2026-03-30T21:11:24.210917Z","end":"2026-03-30T21:11:24.505020Z","steps":["trace[1450384589] 'process raft request' (duration: 293.920746ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:24.510713Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"48.133822ms"} {"level":"warn","ts":"2026-03-30T21:11:24.510744Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"48.166742ms"} {"level":"info","ts":"2026-03-30T21:11:24.654674Z","caller":"traceutil/trace.go:172","msg":"trace[2018708199] linearizableReadLoop","detail":"{readStateIndex:1168; appliedIndex:1169; }","duration":"149.142925ms","start":"2026-03-30T21:11:24.505512Z","end":"2026-03-30T21:11:24.654655Z","steps":["trace[2018708199] 'read index received' (duration: 149.138755ms)","trace[2018708199] 'applied index is now lower than readState.Index' (duration: 2.97µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:24.654782Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"149.257006ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:24.654814Z","caller":"traceutil/trace.go:172","msg":"trace[2008030154] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1005; }","duration":"149.298227ms","start":"2026-03-30T21:11:24.505507Z","end":"2026-03-30T21:11:24.654805Z","steps":["trace[2008030154] 'agreement among raft nodes before linearized reading' (duration: 149.229316ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:24.654783Z","caller":"traceutil/trace.go:172","msg":"trace[604384962] transaction","detail":"{read_only:false; response_revision:1006; number_of_response:1; }","duration":"201.897021ms","start":"2026-03-30T21:11:24.452876Z","end":"2026-03-30T21:11:24.654773Z","steps":["trace[604384962] 'process raft request' (duration: 201.866341ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:25.241801Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"281.902632ms"} {"level":"warn","ts":"2026-03-30T21:11:25.241850Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"281.957773ms"} {"level":"warn","ts":"2026-03-30T21:11:25.369664Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983543,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:25.597760Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"882.439738ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces/kube-system\" limit:1 ","response":"range_response_count:1 size:351"} {"level":"info","ts":"2026-03-30T21:11:25.597839Z","caller":"traceutil/trace.go:172","msg":"trace[1325955610] range","detail":"{range_begin:/registry/namespaces/kube-system; range_end:; response_count:1; response_revision:1006; }","duration":"882.530749ms","start":"2026-03-30T21:11:24.715290Z","end":"2026-03-30T21:11:25.597820Z","steps":["trace[1325955610] 'agreement among raft nodes before linearized reading' (duration: 39.50626ms)","trace[1325955610] 'range keys from in-memory index tree' (duration: 842.776586ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:25.597877Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:24.715275Z","time spent":"882.59392ms","remote":"127.0.0.1:45610","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":1,"response size":374,"request content":"key:\"/registry/namespaces/kube-system\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:25.598281Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"395.710667ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:25.598359Z","caller":"traceutil/trace.go:172","msg":"trace[134912537] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1006; }","duration":"395.801508ms","start":"2026-03-30T21:11:25.202541Z","end":"2026-03-30T21:11:25.598343Z","steps":["trace[134912537] 'range keys from in-memory index tree' (duration: 395.633937ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:25.598745Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"843.885119ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:11:25.598887Z","caller":"traceutil/trace.go:172","msg":"trace[788981643] transaction","detail":"{read_only:false; response_revision:1007; number_of_response:1; }","duration":"896.390904ms","start":"2026-03-30T21:11:24.702483Z","end":"2026-03-30T21:11:25.598874Z","steps":["trace[788981643] 'process raft request' (duration: 52.325582ms)","trace[788981643] 'compare' (duration: 843.525875ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:25.598951Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:24.702456Z","time spent":"896.467335ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:25.840119Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"398.251577ms"} {"level":"warn","ts":"2026-03-30T21:11:25.840230Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"398.366869ms"} {"level":"info","ts":"2026-03-30T21:11:25.840218Z","caller":"traceutil/trace.go:172","msg":"trace[226451036] linearizableReadLoop","detail":"{readStateIndex:1172; appliedIndex:1173; }","duration":"971.135473ms","start":"2026-03-30T21:11:24.869065Z","end":"2026-03-30T21:11:25.840201Z","steps":["trace[226451036] 'read index received' (duration: 971.119522ms)","trace[226451036] 'applied index is now lower than readState.Index' (duration: 12.061µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:25.847991Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"978.906705ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-9hzc6\" limit:1 ","response":"range_response_count:1 size:18349"} {"level":"info","ts":"2026-03-30T21:11:25.848155Z","caller":"traceutil/trace.go:172","msg":"trace[1036160529] range","detail":"{range_begin:/registry/pods/kube-system/cilium-9hzc6; range_end:; response_count:1; response_revision:1008; }","duration":"979.087397ms","start":"2026-03-30T21:11:24.869051Z","end":"2026-03-30T21:11:25.848138Z","steps":["trace[1036160529] 'agreement among raft nodes before linearized reading' (duration: 971.199903ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:25.848365Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:24.869031Z","time spent":"979.3152ms","remote":"127.0.0.1:45740","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18372,"request content":"key:\"/registry/pods/kube-system/cilium-9hzc6\" limit:1 "} {"level":"info","ts":"2026-03-30T21:11:25.848425Z","caller":"traceutil/trace.go:172","msg":"trace[843813894] transaction","detail":"{read_only:false; response_revision:1009; number_of_response:1; }","duration":"965.198882ms","start":"2026-03-30T21:11:24.883201Z","end":"2026-03-30T21:11:25.848400Z","steps":["trace[843813894] 'process raft request' (duration: 956.975544ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:25.848459Z","caller":"traceutil/trace.go:172","msg":"trace[407675064] transaction","detail":"{read_only:false; response_revision:1011; number_of_response:1; }","duration":"948.203059ms","start":"2026-03-30T21:11:24.900235Z","end":"2026-03-30T21:11:25.848438Z","steps":["trace[407675064] 'process raft request' (duration: 948.139689ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:25.848568Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:24.883174Z","time spent":"965.318294ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:25.848604Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:24.900221Z","time spent":"948.30876ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:25.849350Z","caller":"traceutil/trace.go:172","msg":"trace[394653947] transaction","detail":"{read_only:false; response_revision:1010; number_of_response:1; }","duration":"964.111549ms","start":"2026-03-30T21:11:24.885220Z","end":"2026-03-30T21:11:25.849331Z","steps":["trace[394653947] 'process raft request' (duration: 963.104907ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:25.849522Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:24.885202Z","time spent":"964.245012ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:25.855142Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":6530392360376983547,"received-request-id":6530392360376983543} {"level":"warn","ts":"2026-03-30T21:11:25.855920Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"750.410687ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:25.855985Z","caller":"traceutil/trace.go:172","msg":"trace[325534577] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1012; }","duration":"750.464057ms","start":"2026-03-30T21:11:25.105497Z","end":"2026-03-30T21:11:25.855961Z","steps":["trace[325534577] 'agreement among raft nodes before linearized reading' (duration: 750.379166ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:25.856017Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:25.105474Z","time spent":"750.536338ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:11:25.856060Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"251.840807ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/kube-system/cilium-config\" limit:1 ","response":"range_response_count:1 size:6534"} {"level":"info","ts":"2026-03-30T21:11:25.856132Z","caller":"traceutil/trace.go:172","msg":"trace[1962145946] range","detail":"{range_begin:/registry/configmaps/kube-system/cilium-config; range_end:; response_count:1; response_revision:1012; }","duration":"251.966788ms","start":"2026-03-30T21:11:25.604147Z","end":"2026-03-30T21:11:25.856114Z","steps":["trace[1962145946] 'agreement among raft nodes before linearized reading' (duration: 251.770856ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:25.856069Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"257.641865ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:11:25.856241Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"513.794163ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-30T21:11:25.856283Z","caller":"traceutil/trace.go:172","msg":"trace[944226474] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1012; }","duration":"513.839063ms","start":"2026-03-30T21:11:25.342433Z","end":"2026-03-30T21:11:25.856272Z","steps":["trace[944226474] 'agreement among raft nodes before linearized reading' (duration: 513.690121ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:25.856268Z","caller":"traceutil/trace.go:172","msg":"trace[652352018] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1012; }","duration":"257.853878ms","start":"2026-03-30T21:11:25.598397Z","end":"2026-03-30T21:11:25.856251Z","steps":["trace[652352018] 'agreement among raft nodes before linearized reading' (duration: 257.612365ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:25.856325Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:25.342414Z","time spent":"513.902954ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":463,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:26.583207Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"489.176017ms"} {"level":"warn","ts":"2026-03-30T21:11:26.583278Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"489.254969ms"} {"level":"warn","ts":"2026-03-30T21:11:26.673470Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"471.172665ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:26.673884Z","caller":"traceutil/trace.go:172","msg":"trace[1592619345] transaction","detail":"{read_only:false; response_revision:1013; number_of_response:1; }","duration":"814.109405ms","start":"2026-03-30T21:11:25.859742Z","end":"2026-03-30T21:11:26.673851Z","steps":["trace[1592619345] 'process raft request' (duration: 723.554018ms)","trace[1592619345] 'compare' (duration: 89.984871ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:26.674050Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:25.859725Z","time spent":"814.226136ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":426,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:26.674074Z","caller":"traceutil/trace.go:172","msg":"trace[650435441] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1012; }","duration":"471.58636ms","start":"2026-03-30T21:11:26.202275Z","end":"2026-03-30T21:11:26.673861Z","steps":["trace[650435441] 'range keys from in-memory index tree' (duration: 471.143785ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:26.674701Z","caller":"traceutil/trace.go:172","msg":"trace[1953943277] transaction","detail":"{read_only:false; response_revision:1014; number_of_response:1; }","duration":"810.468971ms","start":"2026-03-30T21:11:25.864215Z","end":"2026-03-30T21:11:26.674684Z","steps":["trace[1953943277] 'process raft request' (duration: 810.097716ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:26.674846Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:25.864204Z","time spent":"810.575583ms","remote":"127.0.0.1:45740","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18506,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:27.009423Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983549,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:27.045189Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"261.887414ms"} {"level":"warn","ts":"2026-03-30T21:11:27.045265Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"261.971525ms"} {"level":"info","ts":"2026-03-30T21:11:27.045261Z","caller":"traceutil/trace.go:172","msg":"trace[871610240] linearizableReadLoop","detail":"{readStateIndex:1183; appliedIndex:1183; }","duration":"536.500531ms","start":"2026-03-30T21:11:26.508734Z","end":"2026-03-30T21:11:27.045235Z","steps":["trace[871610240] 'read index received' (duration: 536.490731ms)","trace[871610240] 'applied index is now lower than readState.Index' (duration: 8.34µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:27.095067Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"586.313904ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T21:11:27.095477Z","caller":"traceutil/trace.go:172","msg":"trace[1624168533] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1017; }","duration":"586.726989ms","start":"2026-03-30T21:11:26.508730Z","end":"2026-03-30T21:11:27.095457Z","steps":["trace[1624168533] 'agreement among raft nodes before linearized reading' (duration: 536.652234ms)","trace[1624168533] 'range keys from in-memory index tree' (duration: 49.555859ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:27.095623Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:26.508712Z","time spent":"586.895452ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:27.546445Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983553,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:28.006758Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"761.466147ms"} {"level":"warn","ts":"2026-03-30T21:11:28.006816Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"761.531457ms"} {"level":"warn","ts":"2026-03-30T21:11:28.006815Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":6530392360376983553,"received-request-id":6530392360376983549} {"level":"warn","ts":"2026-03-30T21:11:28.046608Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983553,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:28.546936Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983553,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:29.047225Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376983553,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:11:29.105720Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000580986s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-30T21:11:29.105798Z","caller":"traceutil/trace.go:172","msg":"trace[1350907230] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000693747s","start":"2026-03-30T21:11:27.105087Z","end":"2026-03-30T21:11:29.105781Z","steps":["trace[1350907230] 'agreement among raft nodes before linearized reading' (duration: 2.000577866s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.105918Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:27.105059Z","time spent":"2.000849408s","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/30 21:11:29 WARNING: [core] [Server #4]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-30T21:11:29.123791Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"89ba38f964b95aa0 [logterm: 2, index: 1190, vote: 89ba38f964b95aa0] ignored MsgPreVote from 7ba82bfa2d9da6c2 [logterm: 2, index: 1189] at term 2: lease is not expired (remaining ticks: 3)"} {"level":"warn","ts":"2026-03-30T21:11:29.144038Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.137132296s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-30T21:11:29.144242Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"937.405709ms"} {"level":"warn","ts":"2026-03-30T21:11:29.144271Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"937.439329ms"} {"level":"warn","ts":"2026-03-30T21:11:29.177407Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.082253097s","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T21:11:29.178228Z","caller":"traceutil/trace.go:172","msg":"trace[688341252] linearizableReadLoop","detail":"{readStateIndex:1185; appliedIndex:1184; }","duration":"2.1328876s","start":"2026-03-30T21:11:27.045322Z","end":"2026-03-30T21:11:29.178209Z","steps":["trace[688341252] 'read index received' (duration: 2.099016258s)","trace[688341252] 'applied index is now lower than readState.Index' (duration: 33.870062ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:11:29.178477Z","caller":"traceutil/trace.go:172","msg":"trace[1953215120] transaction","detail":"{read_only:false; response_revision:1022; number_of_response:1; }","duration":"1.323676925s","start":"2026-03-30T21:11:27.854786Z","end":"2026-03-30T21:11:29.178463Z","steps":["trace[1953215120] 'process raft request' (duration: 1.323638943s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.178513Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.504373109s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:29.178539Z","caller":"traceutil/trace.go:172","msg":"trace[701609004] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1018; }","duration":"2.504411259s","start":"2026-03-30T21:11:26.674121Z","end":"2026-03-30T21:11:29.178532Z","steps":["trace[701609004] 'agreement among raft nodes before linearized reading' (duration: 2.504172906s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.178580Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:27.854712Z","time spent":"1.323810146s","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:29.178860Z","caller":"traceutil/trace.go:172","msg":"trace[1339717849] transaction","detail":"{read_only:false; response_revision:1020; number_of_response:1; }","duration":"2.078601794s","start":"2026-03-30T21:11:27.100247Z","end":"2026-03-30T21:11:29.178849Z","steps":["trace[1339717849] 'process raft request' (duration: 2.078092908s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.178929Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:27.100224Z","time spent":"2.078660165s","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:29.179022Z","caller":"traceutil/trace.go:172","msg":"trace[583288783] transaction","detail":"{read_only:false; response_revision:1021; number_of_response:1; }","duration":"1.574736571s","start":"2026-03-30T21:11:27.604277Z","end":"2026-03-30T21:11:29.179014Z","steps":["trace[583288783] 'process raft request' (duration: 1.574116523s)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.179089Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:27.604252Z","time spent":"1.574792221s","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:29.195674Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":6530392360376983557,"received-request-id":6530392360376983553} {"level":"warn","ts":"2026-03-30T21:11:29.196705Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"673.622152ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/priorityclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:29.196759Z","caller":"traceutil/trace.go:172","msg":"trace[371583741] range","detail":"{range_begin:/registry/priorityclasses; range_end:; response_count:0; response_revision:1023; }","duration":"673.676143ms","start":"2026-03-30T21:11:28.523068Z","end":"2026-03-30T21:11:29.196744Z","steps":["trace[371583741] 'agreement among raft nodes before linearized reading' (duration: 673.604182ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.196795Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"517.251062ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:11:29.196879Z","caller":"traceutil/trace.go:172","msg":"trace[655710029] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1023; }","duration":"517.398054ms","start":"2026-03-30T21:11:28.679466Z","end":"2026-03-30T21:11:29.196864Z","steps":["trace[655710029] 'agreement among raft nodes before linearized reading' (duration: 517.146161ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.196933Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:28.679445Z","time spent":"517.474104ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:29.196803Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:28.523054Z","time spent":"673.734893ms","remote":"127.0.0.1:46018","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":28,"request content":"key:\"/registry/priorityclasses\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:29.435316Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"37.93915ms"} {"level":"warn","ts":"2026-03-30T21:11:29.435374Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"38.00064ms"} {"level":"info","ts":"2026-03-30T21:11:29.436572Z","caller":"traceutil/trace.go:172","msg":"trace[866962976] transaction","detail":"{read_only:false; response_revision:1024; number_of_response:1; }","duration":"235.963236ms","start":"2026-03-30T21:11:29.200596Z","end":"2026-03-30T21:11:29.436560Z","steps":["trace[866962976] 'process raft request' (duration: 235.125607ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:29.588755Z","caller":"traceutil/trace.go:172","msg":"trace[883181913] linearizableReadLoop","detail":"{readStateIndex:1190; appliedIndex:1191; }","duration":"385.566675ms","start":"2026-03-30T21:11:29.203169Z","end":"2026-03-30T21:11:29.588736Z","steps":["trace[883181913] 'read index received' (duration: 385.559915ms)","trace[883181913] 'applied index is now lower than readState.Index' (duration: 5.56µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:29.588881Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"385.697628ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:29.588899Z","caller":"traceutil/trace.go:172","msg":"trace[476664275] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1024; }","duration":"385.729528ms","start":"2026-03-30T21:11:29.203164Z","end":"2026-03-30T21:11:29.588893Z","steps":["trace[476664275] 'agreement among raft nodes before linearized reading' (duration: 385.674887ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:29.589128Z","caller":"traceutil/trace.go:172","msg":"trace[228720396] transaction","detail":"{read_only:false; response_revision:1025; number_of_response:1; }","duration":"387.713731ms","start":"2026-03-30T21:11:29.201383Z","end":"2026-03-30T21:11:29.589096Z","steps":["trace[228720396] 'process raft request' (duration: 387.365117ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.589275Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:29.201363Z","time spent":"387.866284ms","remote":"127.0.0.1:45538","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-30T21:11:29.590036Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"383.550952ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:29.590093Z","caller":"traceutil/trace.go:172","msg":"trace[1952884260] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1027; }","duration":"383.613493ms","start":"2026-03-30T21:11:29.206469Z","end":"2026-03-30T21:11:29.590082Z","steps":["trace[1952884260] 'agreement among raft nodes before linearized reading' (duration: 383.527692ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:29.590118Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:29.206459Z","time spent":"383.653823ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-30T21:11:30.991029Z","caller":"traceutil/trace.go:172","msg":"trace[1898171027] transaction","detail":"{read_only:false; response_revision:1055; number_of_response:1; }","duration":"101.209792ms","start":"2026-03-30T21:11:30.889798Z","end":"2026-03-30T21:11:30.991008Z","steps":["trace[1898171027] 'process raft request' (duration: 88.34833ms)","trace[1898171027] 'compare' (duration: 12.752481ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:30.991051Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"102.356027ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-9hzc6\" limit:1 ","response":"range_response_count:1 size:18619"} {"level":"info","ts":"2026-03-30T21:11:30.991117Z","caller":"traceutil/trace.go:172","msg":"trace[878191398] range","detail":"{range_begin:/registry/pods/kube-system/cilium-9hzc6; range_end:; response_count:1; response_revision:1054; }","duration":"102.434868ms","start":"2026-03-30T21:11:30.888667Z","end":"2026-03-30T21:11:30.991102Z","steps":["trace[878191398] 'agreement among raft nodes before linearized reading' (duration: 89.434564ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:31.476081Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"115.379171ms"} {"level":"warn","ts":"2026-03-30T21:11:31.476147Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"115.459431ms"} {"level":"info","ts":"2026-03-30T21:11:31.494680Z","caller":"traceutil/trace.go:172","msg":"trace[704095753] linearizableReadLoop","detail":"{readStateIndex:1230; appliedIndex:1232; }","duration":"309.298618ms","start":"2026-03-30T21:11:31.185367Z","end":"2026-03-30T21:11:31.494666Z","steps":["trace[704095753] 'read index received' (duration: 309.293718ms)","trace[704095753] 'applied index is now lower than readState.Index' (duration: 3.97µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:31.494887Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"309.512441ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:11:31.494916Z","caller":"traceutil/trace.go:172","msg":"trace[21788424] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1062; }","duration":"309.560131ms","start":"2026-03-30T21:11:31.185347Z","end":"2026-03-30T21:11:31.494908Z","steps":["trace[21788424] 'agreement among raft nodes before linearized reading' (duration: 309.403019ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:31.494945Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:31.185321Z","time spent":"309.616072ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T21:11:31.494998Z","caller":"traceutil/trace.go:172","msg":"trace[1585664197] transaction","detail":"{read_only:false; response_revision:1063; number_of_response:1; }","duration":"309.250997ms","start":"2026-03-30T21:11:31.185728Z","end":"2026-03-30T21:11:31.494979Z","steps":["trace[1585664197] 'process raft request' (duration: 309.013364ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:31.495095Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:31.185706Z","time spent":"309.340139ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":494,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:31.495187Z","caller":"traceutil/trace.go:172","msg":"trace[1523096524] transaction","detail":"{read_only:false; response_revision:1062; number_of_response:1; }","duration":"308.962124ms","start":"2026-03-30T21:11:31.185656Z","end":"2026-03-30T21:11:31.494618Z","steps":["trace[1523096524] 'process raft request' (duration: 308.822603ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:31.495668Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"293.185086ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:31.495749Z","caller":"traceutil/trace.go:172","msg":"trace[2031387264] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1065; }","duration":"293.277317ms","start":"2026-03-30T21:11:31.202459Z","end":"2026-03-30T21:11:31.495736Z","steps":["trace[2031387264] 'agreement among raft nodes before linearized reading' (duration: 293.158366ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:31.496388Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:31.185628Z","time spent":"310.692444ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:11:31.601092Z","caller":"traceutil/trace.go:172","msg":"trace[969313755] transaction","detail":"{read_only:false; response_revision:1067; number_of_response:1; }","duration":"101.674809ms","start":"2026-03-30T21:11:31.499390Z","end":"2026-03-30T21:11:31.601072Z","steps":["trace[969313755] 'process raft request' (duration: 100.768418ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:31.714799Z","caller":"traceutil/trace.go:172","msg":"trace[704845419] transaction","detail":"{read_only:false; response_revision:1068; number_of_response:1; }","duration":"100.09921ms","start":"2026-03-30T21:11:31.614683Z","end":"2026-03-30T21:11:31.714782Z","steps":["trace[704845419] 'process raft request' (duration: 98.016016ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:32.025511Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"128.256275ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-9hzc6\" limit:1 ","response":"range_response_count:1 size:18885"} {"level":"info","ts":"2026-03-30T21:11:32.025591Z","caller":"traceutil/trace.go:172","msg":"trace[826999462] range","detail":"{range_begin:/registry/pods/kube-system/cilium-9hzc6; range_end:; response_count:1; response_revision:1073; }","duration":"128.345706ms","start":"2026-03-30T21:11:31.897229Z","end":"2026-03-30T21:11:32.025575Z","steps":["trace[826999462] 'range keys from in-memory index tree' (duration: 127.252912ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:32.229263Z","caller":"traceutil/trace.go:172","msg":"trace[205182146] transaction","detail":"{read_only:false; response_revision:1074; number_of_response:1; }","duration":"330.979376ms","start":"2026-03-30T21:11:31.898264Z","end":"2026-03-30T21:11:32.229243Z","steps":["trace[205182146] 'process raft request' (duration: 330.873764ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:32.229404Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:31.898253Z","time spent":"331.085117ms","remote":"127.0.0.1:45538","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":751,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-30T21:11:32.354461Z","caller":"traceutil/trace.go:172","msg":"trace[1888290427] linearizableReadLoop","detail":"{readStateIndex:1243; appliedIndex:1244; }","duration":"388.47531ms","start":"2026-03-30T21:11:31.965959Z","end":"2026-03-30T21:11:32.354434Z","steps":["trace[1888290427] 'read index received' (duration: 388.466ms)","trace[1888290427] 'applied index is now lower than readState.Index' (duration: 7.37µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:32.713612Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"747.628692ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingress\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:32.713689Z","caller":"traceutil/trace.go:172","msg":"trace[1542355920] range","detail":"{range_begin:/registry/ingress; range_end:; response_count:0; response_revision:1074; }","duration":"747.723462ms","start":"2026-03-30T21:11:31.965947Z","end":"2026-03-30T21:11:32.713670Z","steps":["trace[1542355920] 'agreement among raft nodes before linearized reading' (duration: 388.616472ms)","trace[1542355920] 'range keys from in-memory index tree' (duration: 358.983079ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:32.713723Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:31.965925Z","time spent":"747.790653ms","remote":"127.0.0.1:45886","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":28,"request content":"key:\"/registry/ingress\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:11:32.714108Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"359.511375ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:11:32.714160Z","caller":"traceutil/trace.go:172","msg":"trace[796131762] linearizableReadLoop","detail":"{readStateIndex:1245; appliedIndex:1244; }","duration":"359.619626ms","start":"2026-03-30T21:11:32.354533Z","end":"2026-03-30T21:11:32.714153Z","steps":["trace[796131762] 'read index received' (duration: 827.389µs)","trace[796131762] 'applied index is now lower than readState.Index' (duration: 358.791217ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:32.714340Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"560.770929ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:32.714395Z","caller":"traceutil/trace.go:172","msg":"trace[1798273861] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1075; }","duration":"560.825859ms","start":"2026-03-30T21:11:32.153554Z","end":"2026-03-30T21:11:32.714380Z","steps":["trace[1798273861] 'agreement among raft nodes before linearized reading' (duration: 560.739728ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:32.714430Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:32.153530Z","time spent":"560.8915ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:11:32.714493Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"511.32297ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:32.714510Z","caller":"traceutil/trace.go:172","msg":"trace[446349300] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1075; }","duration":"511.34151ms","start":"2026-03-30T21:11:32.203163Z","end":"2026-03-30T21:11:32.714505Z","steps":["trace[446349300] 'agreement among raft nodes before linearized reading' (duration: 511.31341ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:32.715003Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"234.974954ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattachments\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:32.715029Z","caller":"traceutil/trace.go:172","msg":"trace[370599501] range","detail":"{range_begin:/registry/volumeattachments; range_end:; response_count:0; response_revision:1075; }","duration":"235.001994ms","start":"2026-03-30T21:11:32.480020Z","end":"2026-03-30T21:11:32.715022Z","steps":["trace[370599501] 'agreement among raft nodes before linearized reading' (duration: 234.951134ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:32.715961Z","caller":"traceutil/trace.go:172","msg":"trace[1592928820] transaction","detail":"{read_only:false; response_revision:1075; number_of_response:1; }","duration":"673.384198ms","start":"2026-03-30T21:11:32.040942Z","end":"2026-03-30T21:11:32.714326Z","steps":["trace[1592928820] 'process raft request' (duration: 313.611519ms)","trace[1592928820] 'compare' (duration: 359.365404ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:32.717065Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:32.040920Z","time spent":"675.765415ms","remote":"127.0.0.1:45740","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":19125,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:11:32.985989Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"141.2559ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumeclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:11:32.986095Z","caller":"traceutil/trace.go:172","msg":"trace[903371966] range","detail":"{range_begin:/registry/persistentvolumeclaims; range_end:; response_count:0; response_revision:1075; }","duration":"141.386632ms","start":"2026-03-30T21:11:32.844687Z","end":"2026-03-30T21:11:32.986074Z","steps":["trace[903371966] 'range keys from in-memory index tree' (duration: 140.076235ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:45.712892Z","caller":"traceutil/trace.go:172","msg":"trace[354314076] transaction","detail":"{read_only:false; response_revision:1270; number_of_response:1; }","duration":"141.40579ms","start":"2026-03-30T21:11:45.571465Z","end":"2026-03-30T21:11:45.712870Z","steps":["trace[354314076] 'process raft request' (duration: 141.275689ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:45.715345Z","caller":"traceutil/trace.go:172","msg":"trace[834863202] transaction","detail":"{read_only:false; response_revision:1271; number_of_response:1; }","duration":"138.82985ms","start":"2026-03-30T21:11:45.576496Z","end":"2026-03-30T21:11:45.715326Z","steps":["trace[834863202] 'process raft request' (duration: 138.709249ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:11:48.591321Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60196","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.606354Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60212","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.688635Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60234","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.699348Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60244","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.713752Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60268","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.735401Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60270","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.754579Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60280","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.768349Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60288","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.779012Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60302","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-30T21:11:48.803518Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:60320","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-30T21:11:50.962481Z","caller":"traceutil/trace.go:172","msg":"trace[1097373580] linearizableReadLoop","detail":"{readStateIndex:1549; appliedIndex:1549; }","duration":"117.297524ms","start":"2026-03-30T21:11:50.845161Z","end":"2026-03-30T21:11:50.962459Z","steps":["trace[1097373580] 'read index received' (duration: 117.292054ms)","trace[1097373580] 'applied index is now lower than readState.Index' (duration: 4.35µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:11:50.962717Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"117.531796ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.74\" limit:1 ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-30T21:11:50.962751Z","caller":"traceutil/trace.go:172","msg":"trace[1605827861] range","detail":"{range_begin:/registry/masterleases/199.204.45.74; range_end:; response_count:1; response_revision:1317; }","duration":"117.584577ms","start":"2026-03-30T21:11:50.845157Z","end":"2026-03-30T21:11:50.962741Z","steps":["trace[1605827861] 'agreement among raft nodes before linearized reading' (duration: 117.399765ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:53.850904Z","caller":"traceutil/trace.go:172","msg":"trace[1704053494] transaction","detail":"{read_only:false; response_revision:1335; number_of_response:1; }","duration":"117.701029ms","start":"2026-03-30T21:11:53.733184Z","end":"2026-03-30T21:11:53.850885Z","steps":["trace[1704053494] 'process raft request' (duration: 117.503976ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:53.873257Z","caller":"traceutil/trace.go:172","msg":"trace[1980446929] transaction","detail":"{read_only:false; response_revision:1337; number_of_response:1; }","duration":"126.311901ms","start":"2026-03-30T21:11:53.746922Z","end":"2026-03-30T21:11:53.873234Z","steps":["trace[1980446929] 'process raft request' (duration: 126.25189ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:11:53.873403Z","caller":"traceutil/trace.go:172","msg":"trace[462534928] transaction","detail":"{read_only:false; response_revision:1336; number_of_response:1; }","duration":"127.751908ms","start":"2026-03-30T21:11:53.745632Z","end":"2026-03-30T21:11:53.873384Z","steps":["trace[462534928] 'process raft request' (duration: 127.411134ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:12:00.031765Z","caller":"traceutil/trace.go:172","msg":"trace[1286728614] transaction","detail":"{read_only:false; response_revision:1394; number_of_response:1; }","duration":"135.229176ms","start":"2026-03-30T21:11:59.896518Z","end":"2026-03-30T21:12:00.031748Z","steps":["trace[1286728614] 'process raft request' (duration: 135.102725ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:00.287418Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"55.962361ms"} {"level":"info","ts":"2026-03-30T21:12:00.287479Z","caller":"traceutil/trace.go:172","msg":"trace[2054709552] linearizableReadLoop","detail":"{readStateIndex:1631; appliedIndex:1632; }","duration":"365.511883ms","start":"2026-03-30T21:11:59.921954Z","end":"2026-03-30T21:12:00.287466Z","steps":["trace[2054709552] 'read index received' (duration: 365.508063ms)","trace[2054709552] 'applied index is now lower than readState.Index' (duration: 3.46µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:00.287522Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"56.092913ms"} {"level":"warn","ts":"2026-03-30T21:12:00.871601Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"949.634422ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:12:00.871663Z","caller":"traceutil/trace.go:172","msg":"trace[1513676931] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1394; }","duration":"949.701202ms","start":"2026-03-30T21:11:59.921943Z","end":"2026-03-30T21:12:00.871644Z","steps":["trace[1513676931] 'agreement among raft nodes before linearized reading' (duration: 365.597434ms)","trace[1513676931] 'range keys from in-memory index tree' (duration: 583.952597ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:00.871685Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:59.921930Z","time spent":"949.749183ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:12:00.872489Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"669.245201ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:12:00.872517Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"584.309301ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-30T21:12:00.872563Z","caller":"traceutil/trace.go:172","msg":"trace[1367902636] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1394; }","duration":"670.233133ms","start":"2026-03-30T21:12:00.202316Z","end":"2026-03-30T21:12:00.872549Z","steps":["trace[1367902636] 'range keys from in-memory index tree' (duration: 669.20111ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:00.872662Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"765.965031ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/flowschemas\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:00.872693Z","caller":"traceutil/trace.go:172","msg":"trace[1650903336] range","detail":"{range_begin:/registry/flowschemas; range_end:; response_count:0; response_revision:1395; }","duration":"766.000501ms","start":"2026-03-30T21:12:00.106684Z","end":"2026-03-30T21:12:00.872685Z","steps":["trace[1650903336] 'agreement among raft nodes before linearized reading' (duration: 765.94198ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:12:00.872701Z","caller":"traceutil/trace.go:172","msg":"trace[93895789] transaction","detail":"{read_only:false; response_revision:1395; number_of_response:1; }","duration":"974.547658ms","start":"2026-03-30T21:11:59.898128Z","end":"2026-03-30T21:12:00.872676Z","steps":["trace[93895789] 'process raft request' (duration: 389.326796ms)","trace[93895789] 'compare' (duration: 584.189399ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:00.872722Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:00.106661Z","time spent":"766.053731ms","remote":"127.0.0.1:46146","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/flowschemas\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:12:00.872716Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"766.632788ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:12:00.872799Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:59.898105Z","time spent":"974.65269ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:12:00.872807Z","caller":"traceutil/trace.go:172","msg":"trace[831411648] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1395; }","duration":"766.733939ms","start":"2026-03-30T21:12:00.106055Z","end":"2026-03-30T21:12:00.872789Z","steps":["trace[831411648] 'agreement among raft nodes before linearized reading' (duration: 766.583508ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:00.872973Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:00.106037Z","time spent":"766.924132ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:12:00.872987Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"904.970162ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:12:00.873077Z","caller":"traceutil/trace.go:172","msg":"trace[845143728] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1395; }","duration":"905.081263ms","start":"2026-03-30T21:11:59.967979Z","end":"2026-03-30T21:12:00.873060Z","steps":["trace[845143728] 'agreement among raft nodes before linearized reading' (duration: 904.83605ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:00.873125Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:11:59.967954Z","time spent":"905.161574ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-30T21:12:00.873571Z","caller":"traceutil/trace.go:172","msg":"trace[444790781] linearizableReadLoop","detail":"{readStateIndex:1633; appliedIndex:1632; }","duration":"585.095381ms","start":"2026-03-30T21:12:00.287524Z","end":"2026-03-30T21:12:00.872619Z","steps":["trace[444790781] 'read index received' (duration: 1.307886ms)","trace[444790781] 'applied index is now lower than readState.Index' (duration: 583.785685ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:00.875620Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"214.92205ms"} {"level":"warn","ts":"2026-03-30T21:12:00.875661Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"214.966061ms"} {"level":"warn","ts":"2026-03-30T21:12:00.876662Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"397.423702ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:00.876716Z","caller":"traceutil/trace.go:172","msg":"trace[313324213] range","detail":"{range_begin:/registry/minions; range_end:; response_count:0; response_revision:1396; }","duration":"397.482463ms","start":"2026-03-30T21:12:00.479226Z","end":"2026-03-30T21:12:00.876708Z","steps":["trace[313324213] 'agreement among raft nodes before linearized reading' (duration: 397.398642ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:00.876738Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:00.479208Z","time spent":"397.524824ms","remote":"127.0.0.1:45728","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":28,"request content":"key:\"/registry/minions\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:12:00.879814Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"342.749572ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:184 size:15747"} {"level":"info","ts":"2026-03-30T21:12:00.879875Z","caller":"traceutil/trace.go:172","msg":"trace[1124915960] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:184; response_revision:1396; }","duration":"342.814443ms","start":"2026-03-30T21:12:00.537048Z","end":"2026-03-30T21:12:00.879863Z","steps":["trace[1124915960] 'agreement among raft nodes before linearized reading' (duration: 339.514994ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:00.879912Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:00.537039Z","time spent":"342.860134ms","remote":"127.0.0.1:46320","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":184,"response size":15770,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"warn","ts":"2026-03-30T21:12:34.704524Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376984306,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:12:35.050733Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"700.095409ms"} {"level":"warn","ts":"2026-03-30T21:12:35.050794Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"700.15871ms"} {"level":"info","ts":"2026-03-30T21:12:35.050928Z","caller":"traceutil/trace.go:172","msg":"trace[977038925] transaction","detail":"{read_only:false; response_revision:1547; number_of_response:1; }","duration":"895.226971ms","start":"2026-03-30T21:12:34.155689Z","end":"2026-03-30T21:12:35.050916Z","steps":["trace[977038925] 'process raft request' (duration: 895.13499ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:35.051021Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:34.155665Z","time spent":"895.298972ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:12:35.204775Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":6530392360376984306,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-30T21:12:35.513659Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"262.839547ms"} {"level":"warn","ts":"2026-03-30T21:12:35.513720Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"262.908637ms"} {"level":"info","ts":"2026-03-30T21:12:35.513706Z","caller":"traceutil/trace.go:172","msg":"trace[1522773038] linearizableReadLoop","detail":"{readStateIndex:1805; appliedIndex:1806; }","duration":"1.310566074s","start":"2026-03-30T21:12:34.203116Z","end":"2026-03-30T21:12:35.513682Z","steps":["trace[1522773038] 'read index received' (duration: 1.310557934s)","trace[1522773038] 'applied index is now lower than readState.Index' (duration: 6.79µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:35.520031Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.316892329s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:35.520139Z","caller":"traceutil/trace.go:172","msg":"trace[205932979] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1547; }","duration":"1.317012721s","start":"2026-03-30T21:12:34.203111Z","end":"2026-03-30T21:12:35.520123Z","steps":["trace[205932979] 'agreement among raft nodes before linearized reading' (duration: 1.310675525s)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:12:35.520279Z","caller":"traceutil/trace.go:172","msg":"trace[1314022432] transaction","detail":"{read_only:false; response_revision:1548; number_of_response:1; }","duration":"516.18354ms","start":"2026-03-30T21:12:35.004073Z","end":"2026-03-30T21:12:35.520256Z","steps":["trace[1314022432] 'process raft request' (duration: 509.826644ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:35.521481Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:35.004051Z","time spent":"517.357594ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":493,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:12:35.526183Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":6530392360376984310,"received-request-id":6530392360376984306} {"level":"info","ts":"2026-03-30T21:12:35.526379Z","caller":"traceutil/trace.go:172","msg":"trace[382026440] transaction","detail":"{read_only:false; response_revision:1549; number_of_response:1; }","duration":"133.307064ms","start":"2026-03-30T21:12:35.393058Z","end":"2026-03-30T21:12:35.526365Z","steps":["trace[382026440] 'process raft request' (duration: 133.202882ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:35.527547Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"352.432515ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-30T21:12:35.527593Z","caller":"traceutil/trace.go:172","msg":"trace[479530624] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1549; }","duration":"352.482266ms","start":"2026-03-30T21:12:35.175100Z","end":"2026-03-30T21:12:35.527583Z","steps":["trace[479530624] 'agreement among raft nodes before linearized reading' (duration: 352.346624ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:35.527622Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:35.175077Z","time spent":"352.536786ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":463,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:12:35.527822Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"357.850899ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-30T21:12:35.527854Z","caller":"traceutil/trace.go:172","msg":"trace[1763725891] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1549; }","duration":"357.881089ms","start":"2026-03-30T21:12:35.169962Z","end":"2026-03-30T21:12:35.527843Z","steps":["trace[1763725891] 'agreement among raft nodes before linearized reading' (duration: 357.800759ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:35.527876Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:35.169944Z","time spent":"357.925731ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:12:35.528012Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"424.462451ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:35.528048Z","caller":"traceutil/trace.go:172","msg":"trace[1447542526] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1549; }","duration":"424.499552ms","start":"2026-03-30T21:12:35.103538Z","end":"2026-03-30T21:12:35.528037Z","steps":["trace[1447542526] 'agreement among raft nodes before linearized reading' (duration: 424.449172ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:35.528069Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:35.103518Z","time spent":"424.546242ms","remote":"127.0.0.1:45424","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-30T21:12:35.528302Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"994.683052ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/networkpolicies\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:35.528370Z","caller":"traceutil/trace.go:172","msg":"trace[1952637946] range","detail":"{range_begin:/registry/networkpolicies; range_end:; response_count:0; response_revision:1549; }","duration":"994.755213ms","start":"2026-03-30T21:12:34.533601Z","end":"2026-03-30T21:12:35.528356Z","steps":["trace[1952637946] 'agreement among raft nodes before linearized reading' (duration: 994.650202ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:35.528411Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:34.533575Z","time spent":"994.828074ms","remote":"127.0.0.1:45880","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":28,"request content":"key:\"/registry/networkpolicies\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:12:35.529606Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"750.132738ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:35.529755Z","caller":"traceutil/trace.go:172","msg":"trace[967378911] range","detail":"{range_begin:/registry/leases; range_end:; response_count:0; response_revision:1549; }","duration":"751.19534ms","start":"2026-03-30T21:12:34.778550Z","end":"2026-03-30T21:12:35.529745Z","steps":["trace[967378911] 'agreement among raft nodes before linearized reading' (duration: 749.612641ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:35.530004Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:34.778534Z","time spent":"751.381343ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Range","request count":0,"request size":20,"response count":0,"response size":28,"request content":"key:\"/registry/leases\" limit:1 "} {"level":"warn","ts":"2026-03-30T21:12:45.702498Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"24.784658ms"} {"level":"warn","ts":"2026-03-30T21:12:45.702562Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"24.854679ms"} {"level":"info","ts":"2026-03-30T21:12:45.703182Z","caller":"traceutil/trace.go:172","msg":"trace[1593902060] transaction","detail":"{read_only:false; response_revision:1587; number_of_response:1; }","duration":"144.514665ms","start":"2026-03-30T21:12:45.558427Z","end":"2026-03-30T21:12:45.702941Z","steps":["trace[1593902060] 'process raft request' (duration: 144.202892ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:12:45.736239Z","caller":"traceutil/trace.go:172","msg":"trace[296741825] linearizableReadLoop","detail":"{readStateIndex:1851; appliedIndex:1852; }","duration":"153.395131ms","start":"2026-03-30T21:12:45.582828Z","end":"2026-03-30T21:12:45.736223Z","steps":["trace[296741825] 'read index received' (duration: 153.389421ms)","trace[296741825] 'applied index is now lower than readState.Index' (duration: 4.82µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:45.792648Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"191.444393ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"warn","ts":"2026-03-30T21:12:45.792655Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"209.818902ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:12:45.792711Z","caller":"traceutil/trace.go:172","msg":"trace[124963915] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1587; }","duration":"209.887832ms","start":"2026-03-30T21:12:45.582813Z","end":"2026-03-30T21:12:45.792701Z","steps":["trace[124963915] 'agreement among raft nodes before linearized reading' (duration: 153.479371ms)","trace[124963915] 'range keys from in-memory index tree' (duration: 56.251259ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:12:45.792709Z","caller":"traceutil/trace.go:172","msg":"trace[20827160] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1587; }","duration":"191.516494ms","start":"2026-03-30T21:12:45.601175Z","end":"2026-03-30T21:12:45.792691Z","steps":["trace[20827160] 'agreement among raft nodes before linearized reading' (duration: 136.351479ms)","trace[20827160] 'range keys from in-memory index tree' (duration: 54.995993ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:12:45.792699Z","caller":"traceutil/trace.go:172","msg":"trace[895749734] transaction","detail":"{read_only:false; response_revision:1588; number_of_response:1; }","duration":"231.221435ms","start":"2026-03-30T21:12:45.561452Z","end":"2026-03-30T21:12:45.792673Z","steps":["trace[895749734] 'process raft request' (duration: 175.306361ms)","trace[895749734] 'compare' (duration: 55.720991ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:12:47.280218Z","caller":"traceutil/trace.go:172","msg":"trace[1302445089] transaction","detail":"{read_only:false; response_revision:1593; number_of_response:1; }","duration":"148.430294ms","start":"2026-03-30T21:12:47.131771Z","end":"2026-03-30T21:12:47.280201Z","steps":["trace[1302445089] 'process raft request' (duration: 148.223381ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:51.416472Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"179.482531ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-30T21:12:51.416565Z","caller":"traceutil/trace.go:172","msg":"trace[372941330] linearizableReadLoop","detail":"{readStateIndex:1879; appliedIndex:1878; }","duration":"179.46154ms","start":"2026-03-30T21:12:51.237088Z","end":"2026-03-30T21:12:51.416550Z","steps":["trace[372941330] 'read index received' (duration: 979.291µs)","trace[372941330] 'applied index is now lower than readState.Index' (duration: 178.481349ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:51.416604Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"219.649917ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-30T21:12:51.416662Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"213.92858ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:51.416696Z","caller":"traceutil/trace.go:172","msg":"trace[179774117] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1609; }","duration":"213.96861ms","start":"2026-03-30T21:12:51.202719Z","end":"2026-03-30T21:12:51.416687Z","steps":["trace[179774117] 'agreement among raft nodes before linearized reading' (duration: 213.91199ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:12:51.416703Z","caller":"traceutil/trace.go:172","msg":"trace[505358502] range","detail":"{range_begin:/registry/namespaces; range_end:; response_count:0; response_revision:1609; }","duration":"219.758399ms","start":"2026-03-30T21:12:51.196926Z","end":"2026-03-30T21:12:51.416685Z","steps":["trace[505358502] 'agreement among raft nodes before linearized reading' (duration: 40.206597ms)","trace[505358502] 'range keys from in-memory index tree' (duration: 179.4163ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:51.510957Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"50.677617ms"} {"level":"warn","ts":"2026-03-30T21:12:51.511035Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"50.763038ms"} {"level":"info","ts":"2026-03-30T21:12:51.511253Z","caller":"traceutil/trace.go:172","msg":"trace[1658344352] transaction","detail":"{read_only:false; response_revision:1610; number_of_response:1; }","duration":"219.181282ms","start":"2026-03-30T21:12:51.292050Z","end":"2026-03-30T21:12:51.511231Z","steps":["trace[1658344352] 'process raft request' (duration: 219.029101ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:51.512414Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"122.1352ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/jobs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:51.512464Z","caller":"traceutil/trace.go:172","msg":"trace[2117341805] range","detail":"{range_begin:/registry/jobs; range_end:; response_count:0; response_revision:1610; }","duration":"122.197392ms","start":"2026-03-30T21:12:51.390254Z","end":"2026-03-30T21:12:51.512452Z","steps":["trace[2117341805] 'agreement among raft nodes before linearized reading' (duration: 121.924388ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:56.076399Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"df7880225452eb84","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"215.793568ms"} {"level":"warn","ts":"2026-03-30T21:12:56.076468Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"7ba82bfa2d9da6c2","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"215.877929ms"} {"level":"info","ts":"2026-03-30T21:12:56.076699Z","caller":"traceutil/trace.go:172","msg":"trace[1055733149] transaction","detail":"{read_only:false; response_revision:1626; number_of_response:1; }","duration":"332.515758ms","start":"2026-03-30T21:12:55.744148Z","end":"2026-03-30T21:12:56.076663Z","steps":["trace[1055733149] 'process raft request' (duration: 332.385126ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:56.076846Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:55.744123Z","time spent":"332.65014ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-30T21:12:56.088214Z","caller":"traceutil/trace.go:172","msg":"trace[1573796910] linearizableReadLoop","detail":"{readStateIndex:1896; appliedIndex:1897; }","duration":"188.310155ms","start":"2026-03-30T21:12:55.899883Z","end":"2026-03-30T21:12:56.088193Z","steps":["trace[1573796910] 'read index received' (duration: 188.304945ms)","trace[1573796910] 'applied index is now lower than readState.Index' (duration: 3.99µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:56.170776Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"270.865025ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-30T21:12:56.170858Z","caller":"traceutil/trace.go:172","msg":"trace[526934225] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1626; }","duration":"270.961647ms","start":"2026-03-30T21:12:55.899878Z","end":"2026-03-30T21:12:56.170840Z","steps":["trace[526934225] 'agreement among raft nodes before linearized reading' (duration: 188.411326ms)","trace[526934225] 'range keys from in-memory index tree' (duration: 82.345898ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:12:56.171001Z","caller":"traceutil/trace.go:172","msg":"trace[1140846976] transaction","detail":"{read_only:false; response_revision:1627; number_of_response:1; }","duration":"327.113293ms","start":"2026-03-30T21:12:55.843866Z","end":"2026-03-30T21:12:56.170979Z","steps":["trace[1140846976] 'process raft request' (duration: 244.472872ms)","trace[1140846976] 'compare' (duration: 82.429888ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:56.171159Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-30T21:12:55.843843Z","time spent":"327.245525ms","remote":"127.0.0.1:45862","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":534,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-30T21:12:56.171265Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"271.26249ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-30T21:12:56.171306Z","caller":"traceutil/trace.go:172","msg":"trace[1518176737] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1627; }","duration":"271.30512ms","start":"2026-03-30T21:12:55.899992Z","end":"2026-03-30T21:12:56.171297Z","steps":["trace[1518176737] 'agreement among raft nodes before linearized reading' (duration: 271.205898ms)"],"step_count":1} {"level":"warn","ts":"2026-03-30T21:12:56.210166Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"105.34581ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:56.210282Z","caller":"traceutil/trace.go:172","msg":"trace[1063806576] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1627; }","duration":"105.468372ms","start":"2026-03-30T21:12:56.104793Z","end":"2026-03-30T21:12:56.210261Z","steps":["trace[1063806576] 'agreement among raft nodes before linearized reading' (duration: 67.958756ms)","trace[1063806576] 'range keys from in-memory index tree' (duration: 37.365793ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:56.356902Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"146.813063ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:56.356998Z","caller":"traceutil/trace.go:172","msg":"trace[1074292089] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1629; }","duration":"146.929324ms","start":"2026-03-30T21:12:56.210040Z","end":"2026-03-30T21:12:56.356969Z","steps":["trace[1074292089] 'agreement among raft nodes before linearized reading' (duration: 77.006904ms)","trace[1074292089] 'range keys from in-memory index tree' (duration: 69.791279ms)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:12:56.357045Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"144.7987ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:12:56.357098Z","caller":"traceutil/trace.go:172","msg":"trace[664300140] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1630; }","duration":"144.88155ms","start":"2026-03-30T21:12:56.212205Z","end":"2026-03-30T21:12:56.357087Z","steps":["trace[664300140] 'agreement among raft nodes before linearized reading' (duration: 144.797818ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:13:00.316674Z","caller":"traceutil/trace.go:172","msg":"trace[1524509057] transaction","detail":"{read_only:false; response_revision:1644; number_of_response:1; }","duration":"130.326188ms","start":"2026-03-30T21:13:00.186331Z","end":"2026-03-30T21:13:00.316657Z","steps":["trace[1524509057] 'process raft request' (duration: 130.199636ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:13:00.317199Z","caller":"traceutil/trace.go:172","msg":"trace[539537283] linearizableReadLoop","detail":"{readStateIndex:1917; appliedIndex:1918; }","duration":"113.897422ms","start":"2026-03-30T21:13:00.203276Z","end":"2026-03-30T21:13:00.317174Z","steps":["trace[539537283] 'read index received' (duration: 113.892072ms)","trace[539537283] 'applied index is now lower than readState.Index' (duration: 4.21µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:13:00.317378Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"114.075984ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:13:00.317430Z","caller":"traceutil/trace.go:172","msg":"trace[1493097928] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1644; }","duration":"114.145905ms","start":"2026-03-30T21:13:00.203271Z","end":"2026-03-30T21:13:00.317417Z","steps":["trace[1493097928] 'agreement among raft nodes before linearized reading' (duration: 114.042744ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:13:00.448964Z","caller":"traceutil/trace.go:172","msg":"trace[300003006] transaction","detail":"{read_only:false; response_revision:1646; number_of_response:1; }","duration":"126.180889ms","start":"2026-03-30T21:13:00.322763Z","end":"2026-03-30T21:13:00.448944Z","steps":["trace[300003006] 'process raft request' (duration: 126.128708ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:13:00.450431Z","caller":"traceutil/trace.go:172","msg":"trace[1285571598] transaction","detail":"{read_only:false; response_revision:1645; number_of_response:1; }","duration":"126.30184ms","start":"2026-03-30T21:13:00.322736Z","end":"2026-03-30T21:13:00.449038Z","steps":["trace[1285571598] 'process raft request' (duration: 74.767168ms)","trace[1285571598] 'compare' (duration: 51.255148ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:13:08.679283Z","caller":"traceutil/trace.go:172","msg":"trace[1749162573] transaction","detail":"{read_only:false; response_revision:1678; number_of_response:1; }","duration":"181.401484ms","start":"2026-03-30T21:13:08.497858Z","end":"2026-03-30T21:13:08.679259Z","steps":["trace[1749162573] 'process raft request' (duration: 120.41606ms)","trace[1749162573] 'compare' (duration: 60.883213ms)"],"step_count":2} {"level":"info","ts":"2026-03-30T21:13:08.715918Z","caller":"traceutil/trace.go:172","msg":"trace[1537866965] linearizableReadLoop","detail":"{readStateIndex:1957; appliedIndex:1958; }","duration":"213.437184ms","start":"2026-03-30T21:13:08.502460Z","end":"2026-03-30T21:13:08.715898Z","steps":["trace[1537866965] 'read index received' (duration: 213.429244ms)","trace[1537866965] 'applied index is now lower than readState.Index' (duration: 6.8µs)"],"step_count":2} {"level":"warn","ts":"2026-03-30T21:13:08.716063Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"213.582486ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumloadbalancerippools\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-30T21:13:08.716139Z","caller":"traceutil/trace.go:172","msg":"trace[1421209314] range","detail":"{range_begin:/registry/cilium.io/ciliumloadbalancerippools; range_end:; response_count:0; response_revision:1678; }","duration":"213.664697ms","start":"2026-03-30T21:13:08.502458Z","end":"2026-03-30T21:13:08.716123Z","steps":["trace[1421209314] 'agreement among raft nodes before linearized reading' (duration: 213.549366ms)"],"step_count":1} {"level":"info","ts":"2026-03-30T21:13:08.722292Z","caller":"traceutil/trace.go:172","msg":"trace[1554705346] transaction","detail":"{read_only:false; response_revision:1679; number_of_response:1; }","duration":"224.02805ms","start":"2026-03-30T21:13:08.498250Z","end":"2026-03-30T21:13:08.722279Z","steps":["trace[1554705346] 'process raft request' (duration: 223.930658ms)"],"step_count":1}