{"level":"warn","ts":"2026-03-26T14:49:54.130502Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"warn","ts":"2026-03-26T14:49:54.130691Z","caller":"etcdmain/config.go:270","msg":"--snapshot-count is deprecated in 3.6 and will be decommissioned in 3.7."} {"level":"info","ts":"2026-03-26T14:49:54.130706Z","caller":"etcdmain/etcd.go:64","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.204.45.157:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--feature-gates=InitialCorruptCheck=true","--initial-advertise-peer-urls=https://199.204.45.157:2380","--initial-cluster=controller-1=https://199.204.45.157:2380","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.204.45.157:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.204.45.157:2380","--name=controller-1","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--watch-progress-notify-interval=5s"]} {"level":"info","ts":"2026-03-26T14:49:54.131627Z","caller":"etcdmain/etcd.go:124","msg":"Initialize and start etcd server","data-dir":"/var/lib/etcd","dir-type":"empty"} {"level":"warn","ts":"2026-03-26T14:49:54.131708Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-26T14:49:54.131888Z","caller":"embed/etcd.go:138","msg":"configuring peer listeners","listen-peer-urls":["https://199.204.45.157:2380"]} {"level":"info","ts":"2026-03-26T14:49:54.132087Z","caller":"embed/etcd.go:544","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-26T14:49:54.134670Z","caller":"embed/etcd.go:146","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.157:2379"]} {"level":"info","ts":"2026-03-26T14:49:54.135018Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.6.4","git-sha":"5400cdc","go-version":"go1.23.11","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-1","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.204.45.157:2380"],"listen-peer-urls":["https://199.204.45.157:2380"],"advertise-client-urls":["https://199.204.45.157:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.157:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"experimental-local-address":"","cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://199.204.45.157:2380","initial-cluster-state":"new","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"feature-gates":"InitialCorruptCheck=true","initial-corrupt-check":false,"corrupt-check-time-interval":"0s","compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","discovery-token":"","discovery-endpoints":"","discovery-dial-timeout":"2s","discovery-request-timeout":"5s","discovery-keepalive-time":"2s","discovery-keepalive-timeout":"6s","discovery-insecure-transport":true,"discovery-insecure-skip-tls-verify":false,"discovery-cert":"","discovery-key":"","discovery-cacert":"","discovery-user":"","downgrade-check-interval":"5s","max-learners":1,"v2-deprecation":"write-only"} {"level":"info","ts":"2026-03-26T14:49:54.136554Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc000138188}"} {"level":"info","ts":"2026-03-26T14:49:54.144209Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-26T14:49:54.144446Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"8.075068ms"} {"level":"info","ts":"2026-03-26T14:49:54.144542Z","caller":"etcdserver/bootstrap.go:220","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-03-26T14:49:54.144568Z","caller":"etcdserver/bootstrap.go:94","msg":"bootstrapping cluster"} {"level":"info","ts":"2026-03-26T14:49:54.144640Z","caller":"etcdserver/bootstrap.go:101","msg":"bootstrapping storage"} {"level":"info","ts":"2026-03-26T14:49:54.156164Z","caller":"etcdserver/bootstrap.go:109","msg":"bootstrapping raft"} {"level":"info","ts":"2026-03-26T14:49:54.156250Z","caller":"etcdserver/bootstrap.go:499","msg":"starting local member","local-member-id":"5f170c75a1d97cc","cluster-id":"44a3176ec85feb9"} {"level":"info","ts":"2026-03-26T14:49:54.156396Z","caller":"etcdserver/server.go:312","msg":"bootstrap successfully"} {"level":"info","ts":"2026-03-26T14:49:54.156472Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"5f170c75a1d97cc switched to configuration voters=()"} {"level":"info","ts":"2026-03-26T14:49:54.156519Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"5f170c75a1d97cc became follower at term 0"} {"level":"info","ts":"2026-03-26T14:49:54.156529Z","logger":"raft","caller":"v3@v3.6.0/raft.go:493","msg":"newRaft 5f170c75a1d97cc [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"info","ts":"2026-03-26T14:49:54.156557Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"5f170c75a1d97cc became follower at term 1"} {"level":"info","ts":"2026-03-26T14:49:54.156595Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"5f170c75a1d97cc switched to configuration voters=(428247441089599436)"} {"level":"warn","ts":"2026-03-26T14:49:54.166309Z","caller":"auth/store.go:1135","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-26T14:49:54.168756Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-26T14:49:54.171951Z","caller":"storage/quota.go:93","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-26T14:49:54.172629Z","caller":"etcdserver/server.go:598","msg":"starting etcd server","local-member-id":"5f170c75a1d97cc","local-server-version":"3.6.4","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-26T14:49:54.172864Z","caller":"embed/etcd.go:766","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-26T14:49:54.172886Z","caller":"etcdserver/server.go:483","msg":"started as single-node; fast-forwarding election ticks","local-member-id":"5f170c75a1d97cc","forward-ticks":9,"forward-duration":"900ms","election-ticks":10,"election-timeout":"1s"} {"level":"info","ts":"2026-03-26T14:49:54.173004Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-26T14:49:54.173098Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-26T14:49:54.173122Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-26T14:49:54.173232Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"5f170c75a1d97cc","initial-advertise-peer-urls":["https://199.204.45.157:2380"],"listen-peer-urls":["https://199.204.45.157:2380"],"advertise-client-urls":["https://199.204.45.157:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.157:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-26T14:49:54.173220Z","caller":"embed/etcd.go:640","msg":"serving peer traffic","address":"199.204.45.157:2380"} {"level":"info","ts":"2026-03-26T14:49:54.173298Z","caller":"embed/etcd.go:890","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-26T14:49:54.173660Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"5f170c75a1d97cc switched to configuration voters=(428247441089599436)"} {"level":"info","ts":"2026-03-26T14:49:54.173285Z","caller":"embed/etcd.go:611","msg":"cmux::serve","address":"199.204.45.157:2380"} {"level":"info","ts":"2026-03-26T14:49:54.173958Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"44a3176ec85feb9","local-member-id":"5f170c75a1d97cc","added-peer-id":"5f170c75a1d97cc","added-peer-peer-urls":["https://199.204.45.157:2380"],"added-peer-is-learner":false} {"level":"info","ts":"2026-03-26T14:49:54.457582Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"5f170c75a1d97cc is starting a new election at term 1"} {"level":"info","ts":"2026-03-26T14:49:54.457658Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"5f170c75a1d97cc became pre-candidate at term 1"} {"level":"info","ts":"2026-03-26T14:49:54.457781Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"5f170c75a1d97cc received MsgPreVoteResp from 5f170c75a1d97cc at term 1"} {"level":"info","ts":"2026-03-26T14:49:54.457809Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"5f170c75a1d97cc has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-26T14:49:54.457841Z","logger":"raft","caller":"v3@v3.6.0/raft.go:912","msg":"5f170c75a1d97cc became candidate at term 2"} {"level":"info","ts":"2026-03-26T14:49:54.458616Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"5f170c75a1d97cc received MsgVoteResp from 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:49:54.458645Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"5f170c75a1d97cc has received 1 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-26T14:49:54.458670Z","logger":"raft","caller":"v3@v3.6.0/raft.go:970","msg":"5f170c75a1d97cc became leader at term 2"} {"level":"info","ts":"2026-03-26T14:49:54.458690Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: 5f170c75a1d97cc elected leader 5f170c75a1d97cc at term 2"} {"level":"info","ts":"2026-03-26T14:49:54.459899Z","caller":"etcdserver/server.go:2404","msg":"setting up initial cluster version using v3 API","cluster-version":"3.6"} {"level":"info","ts":"2026-03-26T14:49:54.461356Z","caller":"membership/cluster.go:682","msg":"set initial cluster version","cluster-id":"44a3176ec85feb9","local-member-id":"5f170c75a1d97cc","cluster-version":"3.6"} {"level":"info","ts":"2026-03-26T14:49:54.461435Z","caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.6"} {"level":"info","ts":"2026-03-26T14:49:54.461472Z","caller":"etcdserver/server.go:2424","msg":"cluster version is updated","cluster-version":"3.6"} {"level":"info","ts":"2026-03-26T14:49:54.461509Z","caller":"etcdserver/server.go:1804","msg":"published local member to cluster through raft","local-member-id":"5f170c75a1d97cc","local-member-attributes":"{Name:controller-1 ClientURLs:[https://199.204.45.157:2379]}","cluster-id":"44a3176ec85feb9","publish-timeout":"7s"} {"level":"info","ts":"2026-03-26T14:49:54.461542Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-26T14:49:54.461614Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-26T14:49:54.461761Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-26T14:49:54.461789Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-03-26T14:49:54.461819Z","caller":"version/monitor.go:116","msg":"cluster version differs from storage version.","cluster-version":"3.6.0","storage-version":"3.5.0"} {"level":"info","ts":"2026-03-26T14:49:54.462020Z","caller":"schema/migration.go:65","msg":"updated storage version","new-storage-version":"3.6.0"} {"level":"info","ts":"2026-03-26T14:49:54.463445Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-26T14:49:54.468356Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"warn","ts":"2026-03-26T14:49:54.469630Z","caller":"v3rpc/grpc.go:52","msg":"etcdserver: failed to register grpc metrics","error":"duplicate metrics collector registration attempted"} {"level":"info","ts":"2026-03-26T14:49:54.469822Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-26T14:49:54.482651Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.204.45.157:2379"} {"level":"warn","ts":"2026-03-26T14:49:54.890054Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36524","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.890829Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36538","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.916473Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36590","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.925091Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36620","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.936230Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36646","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.954973Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36660","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.962469Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36682","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.968814Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36702","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.977816Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36712","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.985622Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36726","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:54.996180Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36742","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.006211Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36756","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.015846Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36786","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.024317Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36798","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.035802Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36838","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.044306Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36842","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.051879Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36856","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.059700Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36874","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.069928Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36894","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.077581Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36938","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.084273Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36954","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.093627Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36958","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.134671Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36974","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.143583Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36994","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.159105Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37014","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.168652Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37040","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.180055Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37050","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.191209Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37080","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.199976Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37106","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.209932Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37118","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.221167Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37134","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.233476Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37152","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.242132Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37158","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.253843Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37166","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.262879Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37186","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.273625Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37224","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.283174Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37232","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.293019Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37250","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.305067Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37272","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.316932Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37292","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.324773Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37316","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.336603Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37344","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.344864Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37372","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.353058Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37376","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.363041Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37402","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.370068Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37424","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.379849Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37434","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.396618Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37458","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.408568Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37472","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.417041Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37490","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.425966Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37504","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.437433Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37530","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.445353Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37540","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.453156Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37564","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.459701Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37584","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.467668Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37600","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.478483Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37614","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.489832Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37628","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.507226Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37654","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.554848Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37694","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.564430Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37708","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:49:55.601460Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:37740","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-26T14:49:58.802178Z","caller":"traceutil/trace.go:172","msg":"trace[2104342011] transaction","detail":"{read_only:false; response_revision:222; number_of_response:1; }","duration":"570.428548ms","start":"2026-03-26T14:49:58.231703Z","end":"2026-03-26T14:49:58.802132Z","steps":["trace[2104342011] 'process raft request' (duration: 565.315492ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:49:58.802466Z","caller":"traceutil/trace.go:172","msg":"trace[804879315] transaction","detail":"{read_only:false; response_revision:223; number_of_response:1; }","duration":"483.164665ms","start":"2026-03-26T14:49:58.319198Z","end":"2026-03-26T14:49:58.802363Z","steps":["trace[804879315] 'process raft request' (duration: 482.801242ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:49:58.803064Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:49:58.231678Z","time spent":"570.651589ms","remote":"127.0.0.1:36718","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-26T14:49:58.802631Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:49:58.325491Z","time spent":"477.136091ms","remote":"127.0.0.1:36678","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:49:58.803260Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:49:58.319173Z","time spent":"483.436457ms","remote":"127.0.0.1:37048","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":982,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-26T14:50:06.846146Z","caller":"traceutil/trace.go:172","msg":"trace[746464217] transaction","detail":"{read_only:false; response_revision:411; number_of_response:1; }","duration":"198.809989ms","start":"2026-03-26T14:50:06.647315Z","end":"2026-03-26T14:50:06.846125Z","steps":["trace[746464217] 'process raft request' (duration: 188.456969ms)","trace[746464217] 'compare' (duration: 10.176493ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:09.644170Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"546.606891ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:09.644295Z","caller":"traceutil/trace.go:172","msg":"trace[1296189801] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:422; }","duration":"546.719026ms","start":"2026-03-26T14:50:09.097529Z","end":"2026-03-26T14:50:09.644248Z","steps":["trace[1296189801] 'range keys from in-memory index tree' (duration: 546.556989ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:09.644197Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.177685238s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/etcd-controller-1\" limit:1 ","response":"range_response_count:1 size:5967"} {"level":"info","ts":"2026-03-26T14:50:09.644341Z","caller":"traceutil/trace.go:172","msg":"trace[1165296626] range","detail":"{range_begin:/registry/pods/kube-system/etcd-controller-1; range_end:; response_count:1; response_revision:422; }","duration":"1.177846515s","start":"2026-03-26T14:50:08.466481Z","end":"2026-03-26T14:50:09.644327Z","steps":["trace[1165296626] 'range keys from in-memory index tree' (duration: 1.177544452s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:09.644409Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:08.466455Z","time spent":"1.177945529s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":5989,"request content":"key:\"/registry/pods/kube-system/etcd-controller-1\" limit:1 "} {"level":"info","ts":"2026-03-26T14:50:09.670779Z","caller":"traceutil/trace.go:172","msg":"trace[1259991350] transaction","detail":"{read_only:false; response_revision:423; number_of_response:1; }","duration":"105.784359ms","start":"2026-03-26T14:50:09.564980Z","end":"2026-03-26T14:50:09.670764Z","steps":["trace[1259991350] 'process raft request' (duration: 105.678435ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:09.964200Z","caller":"traceutil/trace.go:172","msg":"trace[305181033] transaction","detail":"{read_only:false; response_revision:424; number_of_response:1; }","duration":"312.167584ms","start":"2026-03-26T14:50:09.652017Z","end":"2026-03-26T14:50:09.964184Z","steps":["trace[305181033] 'process raft request' (duration: 312.059259ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:09.964564Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:09.651908Z","time spent":"312.349082ms","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5780,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:50:30.408231Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"129.698055ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-26T14:50:30.408451Z","caller":"traceutil/trace.go:172","msg":"trace[188934242] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:477; }","duration":"129.927592ms","start":"2026-03-26T14:50:30.278500Z","end":"2026-03-26T14:50:30.408427Z","steps":["trace[188934242] 'agreement among raft nodes before linearized reading' (duration: 129.55465ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:30.407857Z","caller":"traceutil/trace.go:172","msg":"trace[1813740317] linearizableReadLoop","detail":"{readStateIndex:498; appliedIndex:498; }","duration":"129.326822ms","start":"2026-03-26T14:50:30.278512Z","end":"2026-03-26T14:50:30.407839Z","steps":["trace[1813740317] 'read index received' (duration: 129.320202ms)","trace[1813740317] 'applied index is now lower than readState.Index' (duration: 5.67µs)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:50:30.408350Z","caller":"traceutil/trace.go:172","msg":"trace[227574198] transaction","detail":"{read_only:false; response_revision:477; number_of_response:1; }","duration":"167.679836ms","start":"2026-03-26T14:50:30.240365Z","end":"2026-03-26T14:50:30.408045Z","steps":["trace[227574198] 'process raft request' (duration: 167.586733ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:30.433553Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"152.786514ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-26T14:50:30.433634Z","caller":"traceutil/trace.go:172","msg":"trace[753537084] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:477; }","duration":"152.882537ms","start":"2026-03-26T14:50:30.280736Z","end":"2026-03-26T14:50:30.433618Z","steps":["trace[753537084] 'agreement among raft nodes before linearized reading' (duration: 152.65655ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:30.434434Z","caller":"traceutil/trace.go:172","msg":"trace[174503766] transaction","detail":"{read_only:false; response_revision:478; number_of_response:1; }","duration":"188.130379ms","start":"2026-03-26T14:50:30.245675Z","end":"2026-03-26T14:50:30.433806Z","steps":["trace[174503766] 'process raft request' (duration: 187.901762ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:38.690052Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"210.647782ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-26T14:50:38.690354Z","caller":"traceutil/trace.go:172","msg":"trace[996770357] transaction","detail":"{read_only:false; response_revision:497; number_of_response:1; }","duration":"205.739585ms","start":"2026-03-26T14:50:38.484544Z","end":"2026-03-26T14:50:38.690283Z","steps":["trace[996770357] 'process raft request' (duration: 130.656687ms)","trace[996770357] 'compare' (duration: 74.158761ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:50:38.690466Z","caller":"traceutil/trace.go:172","msg":"trace[1499003445] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:496; }","duration":"211.744494ms","start":"2026-03-26T14:50:38.478682Z","end":"2026-03-26T14:50:38.690426Z","steps":["trace[1499003445] 'agreement among raft nodes before linearized reading' (duration: 136.505241ms)","trace[1499003445] 'range keys from in-memory index tree' (duration: 73.960185ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:50:38.716562Z","caller":"traceutil/trace.go:172","msg":"trace[1296021824] linearizableReadLoop","detail":"{readStateIndex:521; appliedIndex:521; }","duration":"101.408594ms","start":"2026-03-26T14:50:38.615135Z","end":"2026-03-26T14:50:38.716543Z","steps":["trace[1296021824] 'read index received' (duration: 101.401784ms)","trace[1296021824] 'applied index is now lower than readState.Index' (duration: 5.78µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:38.716705Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"121.975878ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:38.716749Z","caller":"traceutil/trace.go:172","msg":"trace[1454786007] range","detail":"{range_begin:/registry/pods; range_end:; response_count:0; response_revision:497; }","duration":"122.04358ms","start":"2026-03-26T14:50:38.594695Z","end":"2026-03-26T14:50:38.716738Z","steps":["trace[1454786007] 'agreement among raft nodes before linearized reading' (duration: 121.931457ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:38.716804Z","caller":"traceutil/trace.go:172","msg":"trace[493578950] transaction","detail":"{read_only:false; response_revision:498; number_of_response:1; }","duration":"167.28274ms","start":"2026-03-26T14:50:38.549511Z","end":"2026-03-26T14:50:38.716794Z","steps":["trace[493578950] 'process raft request' (duration: 167.208867ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:39.428457Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"599.802949ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:50:39.428533Z","caller":"traceutil/trace.go:172","msg":"trace[616764740] linearizableReadLoop","detail":"{readStateIndex:523; appliedIndex:522; }","duration":"575.215655ms","start":"2026-03-26T14:50:38.853306Z","end":"2026-03-26T14:50:39.428522Z","steps":["trace[616764740] 'read index received' (duration: 56.591µs)","trace[616764740] 'applied index is now lower than readState.Index' (duration: 575.158354ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:50:39.428597Z","caller":"traceutil/trace.go:172","msg":"trace[1307972783] transaction","detail":"{read_only:false; response_revision:499; number_of_response:1; }","duration":"733.593959ms","start":"2026-03-26T14:50:38.694996Z","end":"2026-03-26T14:50:39.428590Z","steps":["trace[1307972783] 'process raft request' (duration: 133.469261ms)","trace[1307972783] 'compare' (duration: 599.529171ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:39.428638Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:38.694962Z","time spent":"733.657381ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":427,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:50:39.428717Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:38.853279Z","time spent":"575.431152ms","remote":"199.204.45.242:60006","response type":"/etcdserverpb.Cluster/MemberList","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:50:39.428923Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"330.830257ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:39.429830Z","caller":"traceutil/trace.go:172","msg":"trace[519140124] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:499; }","duration":"331.846396ms","start":"2026-03-26T14:50:39.097903Z","end":"2026-03-26T14:50:39.429750Z","steps":["trace[519140124] 'agreement among raft nodes before linearized reading' (duration: 330.785475ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:39.708329Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"5f170c75a1d97cc switched to configuration voters=(428247441089599436) learners=(13087414626088587099)"} {"level":"info","ts":"2026-03-26T14:50:39.708496Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"44a3176ec85feb9","local-member-id":"5f170c75a1d97cc","added-peer-id":"b59fd64328430f5b","added-peer-peer-urls":["https://199.204.45.242:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-26T14:50:39.708516Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:39.708530Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:39.709228Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:39.709409Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b","remote-peer-urls":["https://199.204.45.242:2380"]} {"level":"info","ts":"2026-03-26T14:50:39.709485Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"5f170c75a1d97cc","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:39.709324Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:39.709296Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:39.709359Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:39.709545Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"error","ts":"2026-03-26T14:50:39.765022Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-26T14:50:40.271179Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"warn","ts":"2026-03-26T14:50:41.179846Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"522.884183ms"} {"level":"info","ts":"2026-03-26T14:50:41.179940Z","caller":"traceutil/trace.go:172","msg":"trace[959477214] linearizableReadLoop","detail":"{readStateIndex:524; appliedIndex:524; }","duration":"459.27038ms","start":"2026-03-26T14:50:40.720637Z","end":"2026-03-26T14:50:41.179907Z","steps":["trace[959477214] 'read index received' (duration: 459.258589ms)","trace[959477214] 'applied index is now lower than readState.Index' (duration: 10.261µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:41.180215Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"459.562608ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-26T14:50:41.180257Z","caller":"traceutil/trace.go:172","msg":"trace[758594703] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:499; }","duration":"459.62562ms","start":"2026-03-26T14:50:40.720623Z","end":"2026-03-26T14:50:41.180248Z","steps":["trace[758594703] 'agreement among raft nodes before linearized reading' (duration: 459.418784ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:41.180326Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:40.720609Z","time spent":"459.708222ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":459,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-26T14:50:41.180423Z","caller":"traceutil/trace.go:172","msg":"trace[1053269284] transaction","detail":"{read_only:false; response_revision:500; number_of_response:1; }","duration":"629.220085ms","start":"2026-03-26T14:50:40.551146Z","end":"2026-03-26T14:50:41.180366Z","steps":["trace[1053269284] 'process raft request' (duration: 628.954707ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:41.180573Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:40.551126Z","time spent":"629.37341ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:50:41.512442Z","caller":"traceutil/trace.go:172","msg":"trace[1106965626] linearizableReadLoop","detail":"{readStateIndex:525; appliedIndex:525; }","duration":"332.407347ms","start":"2026-03-26T14:50:41.180012Z","end":"2026-03-26T14:50:41.512419Z","steps":["trace[1106965626] 'read index received' (duration: 332.398187ms)","trace[1106965626] 'applied index is now lower than readState.Index' (duration: 7.98µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:41.512541Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:40.765887Z","time spent":"746.649961ms","remote":"199.204.45.242:60066","response type":"/etcdserverpb.Cluster/MemberList","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:50:41.512535Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"414.927524ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:41.512613Z","caller":"traceutil/trace.go:172","msg":"trace[1352023296] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:500; }","duration":"415.018447ms","start":"2026-03-26T14:50:41.097574Z","end":"2026-03-26T14:50:41.512593Z","steps":["trace[1352023296] 'agreement among raft nodes before linearized reading' (duration: 414.875993ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:41.512922Z","caller":"traceutil/trace.go:172","msg":"trace[500891873] transaction","detail":"{read_only:false; response_revision:501; number_of_response:1; }","duration":"816.960255ms","start":"2026-03-26T14:50:40.695942Z","end":"2026-03-26T14:50:41.512902Z","steps":["trace[500891873] 'process raft request' (duration: 816.769729ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:41.513124Z","caller":"traceutil/trace.go:172","msg":"trace[618403668] transaction","detail":"{read_only:false; response_revision:502; number_of_response:1; }","duration":"811.669091ms","start":"2026-03-26T14:50:40.701444Z","end":"2026-03-26T14:50:41.513114Z","steps":["trace[618403668] 'process raft request' (duration: 811.405153ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:41.513260Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:40.701431Z","time spent":"811.728342ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:50:41.513650Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:40.695915Z","time spent":"817.511601ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":495,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"error","ts":"2026-03-26T14:50:41.514806Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"warn","ts":"2026-03-26T14:50:41.586536Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"206.688665ms"} {"level":"warn","ts":"2026-03-26T14:50:41.656430Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"224.058822ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-26T14:50:41.656527Z","caller":"traceutil/trace.go:172","msg":"trace[301364323] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:502; }","duration":"224.172916ms","start":"2026-03-26T14:50:41.432330Z","end":"2026-03-26T14:50:41.656503Z","steps":["trace[301364323] 'agreement among raft nodes before linearized reading' (duration: 154.314085ms)","trace[301364323] 'range keys from in-memory index tree' (duration: 69.554972ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:50:41.656655Z","caller":"traceutil/trace.go:172","msg":"trace[1805511761] transaction","detail":"{read_only:false; response_revision:503; number_of_response:1; }","duration":"470.941304ms","start":"2026-03-26T14:50:41.185689Z","end":"2026-03-26T14:50:41.656630Z","steps":["trace[1805511761] 'process raft request' (duration: 400.990181ms)","trace[1805511761] 'compare' (duration: 69.750558ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:41.656792Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:41.185660Z","time spent":"471.064198ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"error","ts":"2026-03-26T14:50:41.658994Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"info","ts":"2026-03-26T14:50:41.687152Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:41.687294Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:41.698383Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:41.698647Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"5f170c75a1d97cc","to":"b59fd64328430f5b","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-26T14:50:41.698715Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:41.699005Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"5f170c75a1d97cc","to":"b59fd64328430f5b","stream-type":"stream Message"} {"level":"info","ts":"2026-03-26T14:50:41.699061Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"warn","ts":"2026-03-26T14:50:42.421016Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"564.126576ms"} {"level":"warn","ts":"2026-03-26T14:50:42.421778Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"654.66233ms","expected-duration":"100ms","prefix":"","request":"header: cluster_member_attr_set: > ","response":""} {"level":"info","ts":"2026-03-26T14:50:42.422115Z","caller":"traceutil/trace.go:172","msg":"trace[344818376] linearizableReadLoop","detail":"{readStateIndex:530; appliedIndex:529; }","duration":"323.723712ms","start":"2026-03-26T14:50:42.098368Z","end":"2026-03-26T14:50:42.422092Z","steps":["trace[344818376] 'read index received' (duration: 322.801815ms)","trace[344818376] 'applied index is now lower than readState.Index' (duration: 920.357µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:42.422247Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"323.905167ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:42.422322Z","caller":"traceutil/trace.go:172","msg":"trace[714100606] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:504; }","duration":"323.99153ms","start":"2026-03-26T14:50:42.098315Z","end":"2026-03-26T14:50:42.422307Z","steps":["trace[714100606] 'agreement among raft nodes before linearized reading' (duration: 323.873307ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:42.422348Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:41.767227Z","time spent":"655.116582ms","remote":"199.204.45.242:60080","response type":"/etcdserverpb.Cluster/MemberList","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"error","ts":"2026-03-26T14:50:42.424077Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-26T14:50:42.439506Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-26T14:50:42.775552Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"warn","ts":"2026-03-26T14:50:42.791846Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"370.702758ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:42.791925Z","caller":"traceutil/trace.go:172","msg":"trace[647476253] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:504; }","duration":"370.793961ms","start":"2026-03-26T14:50:42.421114Z","end":"2026-03-26T14:50:42.791908Z","steps":["trace[647476253] 'range keys from in-memory index tree' (duration: 369.235775ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:42.791967Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:41.891134Z","time spent":"900.824095ms","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:50:42.792480Z","caller":"etcdserver/server.go:1838","msg":"sending merged snapshot","from":"5f170c75a1d97cc","to":"b59fd64328430f5b","bytes":811011,"size":"811 kB"} {"level":"info","ts":"2026-03-26T14:50:42.792743Z","caller":"rafthttp/snapshot_sender.go:82","msg":"sending database snapshot","snapshot-index":530,"remote-peer-id":"b59fd64328430f5b","bytes":811011,"size":"811 kB"} {"level":"info","ts":"2026-03-26T14:50:42.796218Z","caller":"etcdserver/snapshot_merge.go:64","msg":"sent database snapshot to writer","bytes":802816,"size":"803 kB"} {"level":"info","ts":"2026-03-26T14:50:42.808207Z","caller":"rafthttp/snapshot_sender.go:131","msg":"sent database snapshot","snapshot-index":530,"remote-peer-id":"b59fd64328430f5b","bytes":811011,"size":"811 kB"} {"level":"warn","ts":"2026-03-26T14:50:42.825219Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:50:42.825278Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b","error":"EOF"} {"level":"info","ts":"2026-03-26T14:50:42.836914Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"5f170c75a1d97cc","to":"b59fd64328430f5b","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-26T14:50:42.836959Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:42.836972Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:42.837463Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:42.837708Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:42.840568Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"5f170c75a1d97cc","to":"b59fd64328430f5b","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-26T14:50:42.840598Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:42.840609Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:43.278022Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"5f170c75a1d97cc switched to configuration voters=(428247441089599436 13087414626088587099)"} {"level":"info","ts":"2026-03-26T14:50:43.278139Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"44a3176ec85feb9","local-member-id":"5f170c75a1d97cc","promoted-member-id":"b59fd64328430f5b"} {"level":"info","ts":"2026-03-26T14:50:43.278167Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"5f170c75a1d97cc","raft-conf-change":"ConfChangeAddNode","raft-conf-change-node-id":"b59fd64328430f5b"} {"level":"warn","ts":"2026-03-26T14:50:46.046899Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"153.943938ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:46.047030Z","caller":"traceutil/trace.go:172","msg":"trace[731516044] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:510; }","duration":"154.078512ms","start":"2026-03-26T14:50:45.892913Z","end":"2026-03-26T14:50:46.046992Z","steps":["trace[731516044] 'range keys from in-memory index tree' (duration: 152.741864ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:46.047075Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"299.294919ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-03-26T14:50:46.047171Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"358.181137ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-26T14:50:46.047256Z","caller":"traceutil/trace.go:172","msg":"trace[2028830552] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:510; }","duration":"299.456274ms","start":"2026-03-26T14:50:45.747728Z","end":"2026-03-26T14:50:46.047184Z","steps":["trace[2028830552] 'range keys from in-memory index tree' (duration: 297.651103ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:46.047274Z","caller":"traceutil/trace.go:172","msg":"trace[681862072] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:510; }","duration":"358.312041ms","start":"2026-03-26T14:50:45.688945Z","end":"2026-03-26T14:50:46.047257Z","steps":["trace[681862072] 'range keys from in-memory index tree' (duration: 356.577902ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:46.047331Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:50:45.688922Z","time spent":"358.398043ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":459,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-26T14:50:46.189930Z","caller":"traceutil/trace.go:172","msg":"trace[58326461] transaction","detail":"{read_only:false; response_revision:512; number_of_response:1; }","duration":"135.422378ms","start":"2026-03-26T14:50:46.054477Z","end":"2026-03-26T14:50:46.189899Z","steps":["trace[58326461] 'process raft request' (duration: 115.452579ms)","trace[58326461] 'compare' (duration: 19.795865ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:48.383018Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"186.828185ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-26T14:50:48.383125Z","caller":"traceutil/trace.go:172","msg":"trace[2077203689] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:517; }","duration":"186.946468ms","start":"2026-03-26T14:50:48.196156Z","end":"2026-03-26T14:50:48.383103Z","steps":["trace[2077203689] 'range keys from in-memory index tree' (duration: 184.880712ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:48.558835Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.273715ms"} {"level":"info","ts":"2026-03-26T14:50:48.559050Z","caller":"traceutil/trace.go:172","msg":"trace[796519952] transaction","detail":"{read_only:false; response_revision:518; number_of_response:1; }","duration":"171.594173ms","start":"2026-03-26T14:50:48.387443Z","end":"2026-03-26T14:50:48.559037Z","steps":["trace[796519952] 'process raft request' (duration: 171.499261ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:51.418181Z","caller":"traceutil/trace.go:172","msg":"trace[1022085959] transaction","detail":"{read_only:false; response_revision:524; number_of_response:1; }","duration":"110.775777ms","start":"2026-03-26T14:50:51.307385Z","end":"2026-03-26T14:50:51.418160Z","steps":["trace[1022085959] 'process raft request' (duration: 110.641094ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:51.753140Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"148.899817ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:50:51.753958Z","caller":"traceutil/trace.go:172","msg":"trace[791584216] transaction","detail":"{read_only:false; response_revision:528; number_of_response:1; }","duration":"217.264172ms","start":"2026-03-26T14:50:51.536615Z","end":"2026-03-26T14:50:51.753879Z","steps":["trace[791584216] 'process raft request' (duration: 67.550223ms)","trace[791584216] 'compare' (duration: 148.780944ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:50:56.750820Z","caller":"traceutil/trace.go:172","msg":"trace[119348416] transaction","detail":"{read_only:false; response_revision:539; number_of_response:1; }","duration":"110.744373ms","start":"2026-03-26T14:50:56.640060Z","end":"2026-03-26T14:50:56.750804Z","steps":["trace[119348416] 'process raft request' (duration: 109.995215ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:58.876750Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"100.015369ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:58.876782Z","caller":"traceutil/trace.go:172","msg":"trace[1970865926] transaction","detail":"{read_only:false; response_revision:548; number_of_response:1; }","duration":"116.374731ms","start":"2026-03-26T14:50:58.760388Z","end":"2026-03-26T14:50:58.876762Z","steps":["trace[1970865926] 'process raft request' (duration: 116.234658ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:58.876812Z","caller":"traceutil/trace.go:172","msg":"trace[1825274446] range","detail":"{range_begin:/registry/minions/controller-2; range_end:; response_count:0; response_revision:547; }","duration":"100.096591ms","start":"2026-03-26T14:50:58.776701Z","end":"2026-03-26T14:50:58.876798Z","steps":["trace[1825274446] 'agreement among raft nodes before linearized reading' (duration: 99.862956ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:50:59.615932Z","caller":"traceutil/trace.go:172","msg":"trace[1218485507] linearizableReadLoop","detail":"{readStateIndex:581; appliedIndex:582; }","duration":"143.089858ms","start":"2026-03-26T14:50:59.472810Z","end":"2026-03-26T14:50:59.615900Z","steps":["trace[1218485507] 'read index received' (duration: 143.069248ms)","trace[1218485507] 'applied index is now lower than readState.Index' (duration: 19.23µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:50:59.660383Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"187.558052ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/daemonsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:59.660463Z","caller":"traceutil/trace.go:172","msg":"trace[1780423033] range","detail":"{range_begin:/registry/daemonsets; range_end:; response_count:0; response_revision:549; }","duration":"187.649554ms","start":"2026-03-26T14:50:59.472794Z","end":"2026-03-26T14:50:59.660443Z","steps":["trace[1780423033] 'agreement among raft nodes before linearized reading' (duration: 143.16932ms)","trace[1780423033] 'range keys from in-memory index tree' (duration: 44.347291ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:50:59.660553Z","caller":"traceutil/trace.go:172","msg":"trace[1743104600] transaction","detail":"{read_only:false; response_revision:550; number_of_response:1; }","duration":"173.148561ms","start":"2026-03-26T14:50:59.487377Z","end":"2026-03-26T14:50:59.660525Z","steps":["trace[1743104600] 'process raft request' (duration: 128.488262ms)","trace[1743104600] 'compare' (duration: 44.471444ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:50:59.660747Z","caller":"traceutil/trace.go:172","msg":"trace[568010996] transaction","detail":"{read_only:false; response_revision:551; number_of_response:1; }","duration":"105.209004ms","start":"2026-03-26T14:50:59.555524Z","end":"2026-03-26T14:50:59.660733Z","steps":["trace[568010996] 'process raft request' (duration: 104.944327ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:50:59.660922Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"104.304181ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/limitranges\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:50:59.661536Z","caller":"traceutil/trace.go:172","msg":"trace[1886445174] range","detail":"{range_begin:/registry/limitranges; range_end:; response_count:0; response_revision:551; }","duration":"105.139842ms","start":"2026-03-26T14:50:59.556250Z","end":"2026-03-26T14:50:59.661390Z","steps":["trace[1886445174] 'agreement among raft nodes before linearized reading' (duration: 104.278761ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:01.674104Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"warn","ts":"2026-03-26T14:51:09.967750Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"110.503469ms"} {"level":"info","ts":"2026-03-26T14:51:09.968368Z","caller":"traceutil/trace.go:172","msg":"trace[1550096011] transaction","detail":"{read_only:false; response_revision:601; number_of_response:1; }","duration":"227.094355ms","start":"2026-03-26T14:51:09.741162Z","end":"2026-03-26T14:51:09.968256Z","steps":["trace[1550096011] 'process raft request' (duration: 226.835549ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:10.292908Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:10.793749Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:10.969855Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 643, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:11.294783Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:11.796492Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:11.892615Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000010619s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:11.892792Z","caller":"traceutil/trace.go:172","msg":"trace[703920121] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000241355s","start":"2026-03-26T14:51:09.892530Z","end":"2026-03-26T14:51:11.892772Z","steps":["trace[703920121] 'agreement among raft nodes before linearized reading' (duration: 2.000007549s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:11.893012Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:09.892501Z","time spent":"2.000497641s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:11 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:51:11.970153Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 643, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:12.297474Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:12.798599Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:12.809106Z","caller":"etcdserver/server.go:1856","msg":"sent merged snapshot","from":"5f170c75a1d97cc","to":"b59fd64328430f5b","bytes":811011,"size":"811 kB","took":"30.016631132s"} {"level":"info","ts":"2026-03-26T14:51:12.969653Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 645, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:13.299792Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:13.799992Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:13.893090Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999860247s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:13.893213Z","caller":"traceutil/trace.go:172","msg":"trace[1851142302] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.99998504s","start":"2026-03-26T14:51:11.893193Z","end":"2026-03-26T14:51:13.893178Z","steps":["trace[1851142302] 'agreement among raft nodes before linearized reading' (duration: 1.999855137s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:13.893347Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.893150Z","time spent":"2.000120273s","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:13 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:51:13.974410Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 646, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:14.300609Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:14.801088Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:14.969100Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 646, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:14.980888Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:09.982567Z","time spent":"4.998301454s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} 2026/03/26 14:51:14 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:15.248600Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000042015s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:15.248693Z","caller":"traceutil/trace.go:172","msg":"trace[1103847203] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000145618s","start":"2026-03-26T14:51:13.248530Z","end":"2026-03-26T14:51:15.248676Z","steps":["trace[1103847203] 'agreement among raft nodes before linearized reading' (duration: 2.000040025s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:15.248754Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:13.248494Z","time spent":"2.00024835s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:15 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:15.301271Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:15.801596Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:15.903272Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000202143s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:15.903383Z","caller":"traceutil/trace.go:172","msg":"trace[762283462] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000342016s","start":"2026-03-26T14:51:13.903022Z","end":"2026-03-26T14:51:15.903364Z","steps":["trace[762283462] 'agreement among raft nodes before linearized reading' (duration: 2.000197313s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:15.903441Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:13.903001Z","time spent":"2.000431118s","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:51:15.970543Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 646, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:16.301757Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486915,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:16.792549Z","caller":"etcdserver/v3_server.go:923","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"} {"level":"warn","ts":"2026-03-26T14:51:16.792743Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.000372766s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/jobs\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:16.792784Z","caller":"traceutil/trace.go:172","msg":"trace[585665241] range","detail":"{range_begin:/registry/jobs; range_end:; }","duration":"7.000437007s","start":"2026-03-26T14:51:09.792330Z","end":"2026-03-26T14:51:16.792767Z","steps":["trace[585665241] 'agreement among raft nodes before linearized reading' (duration: 7.000371566s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:16.792836Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:09.792271Z","time spent":"7.000554939s","remote":"127.0.0.1:37004","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/jobs\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:16.969352Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 647, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:16.973655Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.974547Z","time spent":"4.999101473s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} 2026/03/26 14:51:16 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:17.292814Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:17.793603Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:17.912701Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999784194s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:17.912795Z","caller":"traceutil/trace.go:172","msg":"trace[1407493427] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.999893137s","start":"2026-03-26T14:51:15.912884Z","end":"2026-03-26T14:51:17.912777Z","steps":["trace[1407493427] 'agreement among raft nodes before linearized reading' (duration: 1.999780653s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:17.912856Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:15.912861Z","time spent":"1.999976448s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:17 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:51:17.969522Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 648, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:18.294287Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:18.795469Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:18.969417Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 648, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:19.099431Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:12.098738Z","time spent":"7.000686598s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:19.296562Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:19.797387Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:19.913409Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.00027833s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:19.913510Z","caller":"traceutil/trace.go:172","msg":"trace[1984703394] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000390513s","start":"2026-03-26T14:51:17.913100Z","end":"2026-03-26T14:51:19.913491Z","steps":["trace[1984703394] 'agreement among raft nodes before linearized reading' (duration: 2.0002753s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:19.913576Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:17.913086Z","time spent":"2.000480954s","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:51:19.969325Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 649, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:19.981966Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999493042s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:19.982050Z","caller":"traceutil/trace.go:172","msg":"trace[1663685706] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"4.999603125s","start":"2026-03-26T14:51:14.982430Z","end":"2026-03-26T14:51:19.982033Z","steps":["trace[1663685706] 'agreement among raft nodes before linearized reading' (duration: 4.999488252s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:19.982118Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:14.982404Z","time spent":"4.999699786s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:20.167225Z","caller":"etcdserver/server.go:939","msg":"Failed to check current member's leadership","error":"context deadline exceeded"} {"level":"warn","ts":"2026-03-26T14:51:20.167315Z","caller":"etcdserver/server.go:879","msg":"Ignore the lease revoking request because current member isn't a leader","local-member-id":428247441089599436} {"level":"warn","ts":"2026-03-26T14:51:20.297803Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:20.510805Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"9.999353029s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:20.511649Z","caller":"traceutil/trace.go:172","msg":"trace[2101428710] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; }","duration":"9.999448431s","start":"2026-03-26T14:51:10.511420Z","end":"2026-03-26T14:51:20.510868Z","steps":["trace[2101428710] 'agreement among raft nodes before linearized reading' (duration: 9.999349979s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:20.511913Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.511394Z","time spent":"10.000482753s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} 2026/03/26 14:51:20 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:20.797980Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:20.898685Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:13.897776Z","time spent":"7.000902762s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"info","ts":"2026-03-26T14:51:20.969195Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 651, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:20.999465Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"9.99859636s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:20.999654Z","caller":"traceutil/trace.go:172","msg":"trace[634787657] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; }","duration":"9.998756513s","start":"2026-03-26T14:51:11.000760Z","end":"2026-03-26T14:51:20.999516Z","steps":["trace[634787657] 'agreement among raft nodes before linearized reading' (duration: 9.99859239s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:20.999742Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.000733Z","time spent":"9.998971708s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} 2026/03/26 14:51:20 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:21.298442Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:21.799527Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:21.922504Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.0006641s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:21.922597Z","caller":"traceutil/trace.go:172","msg":"trace[556733355] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000769002s","start":"2026-03-26T14:51:19.921809Z","end":"2026-03-26T14:51:21.922578Z","steps":["trace[556733355] 'agreement among raft nodes before linearized reading' (duration: 2.00064895s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:21.922651Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:19.921796Z","time spent":"2.000847333s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:21 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:51:21.970905Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 652, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:21.973903Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.998521243s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:21.973994Z","caller":"traceutil/trace.go:172","msg":"trace[1165903690] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.998631065s","start":"2026-03-26T14:51:16.975344Z","end":"2026-03-26T14:51:21.973975Z","steps":["trace[1165903690] 'agreement among raft nodes before linearized reading' (duration: 4.998517583s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:21.974047Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:16.975325Z","time spent":"4.998712318s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} 2026/03/26 14:51:21 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:22.300370Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:22.801458Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:22.969456Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 652, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:23.302338Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486923,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:23.312589Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:16.311860Z","time spent":"7.000724638s","remote":"127.0.0.1:37642","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:23.666907Z","caller":"etcdserver/server.go:939","msg":"Failed to check current member's leadership","error":"context deadline exceeded"} {"level":"warn","ts":"2026-03-26T14:51:23.666976Z","caller":"etcdserver/server.go:879","msg":"Ignore the lease revoking request because current member isn't a leader","local-member-id":428247441089599436} {"level":"warn","ts":"2026-03-26T14:51:23.793743Z","caller":"etcdserver/v3_server.go:923","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"} {"level":"warn","ts":"2026-03-26T14:51:23.793887Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"11.731431696s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaims\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.793880Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"11.174791828s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaimtemplates\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:23.793940Z","caller":"traceutil/trace.go:172","msg":"trace[2018083913] range","detail":"{range_begin:/registry/resourceclaims; range_end:; }","duration":"11.731501358s","start":"2026-03-26T14:51:12.062423Z","end":"2026-03-26T14:51:23.793924Z","steps":["trace[2018083913] 'agreement among raft nodes before linearized reading' (duration: 11.731430586s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:23.793974Z","caller":"traceutil/trace.go:172","msg":"trace[1222290276] range","detail":"{range_begin:/registry/resourceclaimtemplates; range_end:; }","duration":"11.174909141s","start":"2026-03-26T14:51:12.619046Z","end":"2026-03-26T14:51:23.793956Z","steps":["trace[1222290276] 'agreement among raft nodes before linearized reading' (duration: 11.174794159s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.793994Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:12.062397Z","time spent":"11.731586289s","remote":"127.0.0.1:37676","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":0,"request content":"key:\"/registry/resourceclaims\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794026Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"12.987187746s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.794037Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:12.619002Z","time spent":"11.175024574s","remote":"127.0.0.1:37684","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":0,"request content":"key:\"/registry/resourceclaimtemplates\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794057Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.856425352s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.794066Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"13.673353575s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingwebhookconfigurations\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:23.794073Z","caller":"traceutil/trace.go:172","msg":"trace[109025359] range","detail":"{range_begin:/registry/resourceslices; range_end:; }","duration":"12.987227437s","start":"2026-03-26T14:51:10.806829Z","end":"2026-03-26T14:51:23.794057Z","steps":["trace[109025359] 'agreement among raft nodes before linearized reading' (duration: 12.987188496s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:23.794093Z","caller":"traceutil/trace.go:172","msg":"trace[423682015] range","detail":"{range_begin:/registry/validatingwebhookconfigurations; range_end:; }","duration":"13.673378886s","start":"2026-03-26T14:51:10.120707Z","end":"2026-03-26T14:51:23.794086Z","steps":["trace[423682015] 'agreement among raft nodes before linearized reading' (duration: 13.673353455s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794094Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"9.118340764s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourcequotas\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.794107Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"12.325064845s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/priorityclasses\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.794120Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.120623Z","time spent":"13.673490578s","remote":"127.0.0.1:37556","response type":"/etcdserverpb.KV/Range","request count":0,"request size":45,"response count":0,"response size":0,"request content":"key:\"/registry/validatingwebhookconfigurations\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794034Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.256648327s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaims\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.794099Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"13.695899286s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.794188Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"10.096299726s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:23.794217Z","caller":"traceutil/trace.go:172","msg":"trace[1962147689] range","detail":"{range_begin:/registry/services/specs; range_end:; }","duration":"10.096326116s","start":"2026-03-26T14:51:13.697883Z","end":"2026-03-26T14:51:23.794209Z","steps":["trace[1962147689] 'agreement among raft nodes before linearized reading' (duration: 10.096299626s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:23.794226Z","caller":"traceutil/trace.go:172","msg":"trace[501412889] range","detail":"{range_begin:; range_end:; }","duration":"13.696022818s","start":"2026-03-26T14:51:10.098181Z","end":"2026-03-26T14:51:23.794204Z","steps":["trace[501412889] 'agreement among raft nodes before linearized reading' (duration: 13.695897776s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794285Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:13.697850Z","time spent":"10.096382858s","remote":"127.0.0.1:36924","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":0,"request content":"key:\"/registry/services/specs\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:23.794237Z","caller":"traceutil/trace.go:172","msg":"trace[252824160] range","detail":"{range_begin:/registry/resourceclaims; range_end:; }","duration":"7.2568279s","start":"2026-03-26T14:51:16.537373Z","end":"2026-03-26T14:51:23.794201Z","steps":["trace[252824160] 'agreement among raft nodes before linearized reading' (duration: 7.256648157s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794045Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"13.429617723s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rolebindings\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:23.794370Z","caller":"traceutil/trace.go:172","msg":"trace[1759544423] range","detail":"{range_begin:/registry/rolebindings; range_end:; }","duration":"13.429940749s","start":"2026-03-26T14:51:10.364421Z","end":"2026-03-26T14:51:23.794362Z","steps":["trace[1759544423] 'agreement among raft nodes before linearized reading' (duration: 13.429618183s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794409Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.364391Z","time spent":"13.430008351s","remote":"127.0.0.1:37242","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":0,"request content":"key:\"/registry/rolebindings\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794055Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"12.757069521s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllerrevisions\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:23.794446Z","caller":"traceutil/trace.go:172","msg":"trace[344190780] range","detail":"{range_begin:/registry/controllerrevisions; range_end:; }","duration":"12.7574951s","start":"2026-03-26T14:51:11.036944Z","end":"2026-03-26T14:51:23.794440Z","steps":["trace[344190780] 'agreement among raft nodes before linearized reading' (duration: 12.757065181s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794469Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.036926Z","time spent":"12.75753667s","remote":"127.0.0.1:37544","response type":"/etcdserverpb.KV/Range","request count":0,"request size":33,"response count":0,"response size":0,"request content":"key:\"/registry/controllerrevisions\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.793956Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"13.236143364s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterroles\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.794430Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:16.537351Z","time spent":"7.257045165s","remote":"127.0.0.1:37676","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":0,"request content":"key:\"/registry/resourceclaims\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:23.794506Z","caller":"traceutil/trace.go:172","msg":"trace[1206916414] range","detail":"{range_begin:/registry/clusterroles; range_end:; }","duration":"13.236728276s","start":"2026-03-26T14:51:10.557768Z","end":"2026-03-26T14:51:23.794496Z","steps":["trace[1206916414] 'agreement among raft nodes before linearized reading' (duration: 13.236139254s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.793974Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"9.00677696s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingress\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:23.794092Z","caller":"traceutil/trace.go:172","msg":"trace[1882249798] range","detail":"{range_begin:/registry/secrets; range_end:; }","duration":"7.856453973s","start":"2026-03-26T14:51:15.937626Z","end":"2026-03-26T14:51:23.794080Z","steps":["trace[1882249798] 'agreement among raft nodes before linearized reading' (duration: 7.856425612s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794550Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.557742Z","time spent":"13.236792568s","remote":"127.0.0.1:37260","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":0,"request content":"key:\"/registry/clusterroles\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:23.794546Z","caller":"traceutil/trace.go:172","msg":"trace[499199280] range","detail":"{range_begin:/registry/ingress; range_end:; }","duration":"9.007361732s","start":"2026-03-26T14:51:14.787175Z","end":"2026-03-26T14:51:23.794537Z","steps":["trace[499199280] 'agreement among raft nodes before linearized reading' (duration: 9.00677434s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:23.794152Z","caller":"traceutil/trace.go:172","msg":"trace[1049004256] range","detail":"{range_begin:/registry/priorityclasses; range_end:; }","duration":"12.325100086s","start":"2026-03-26T14:51:11.469033Z","end":"2026-03-26T14:51:23.794133Z","steps":["trace[1049004256] 'agreement among raft nodes before linearized reading' (duration: 12.325065665s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794588Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:14.787152Z","time spent":"9.007426212s","remote":"127.0.0.1:37116","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":0,"request content":"key:\"/registry/ingress\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794116Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.543287319s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ipaddresses\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"warn","ts":"2026-03-26T14:51:23.794603Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:15.937604Z","time spent":"7.856975752s","remote":"127.0.0.1:36718","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":0,"request content":"key:\"/registry/secrets\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:23.794621Z","caller":"traceutil/trace.go:172","msg":"trace[2115038364] range","detail":"{range_begin:/registry/ipaddresses; range_end:; }","duration":"8.543791959s","start":"2026-03-26T14:51:15.250822Z","end":"2026-03-26T14:51:23.794614Z","steps":["trace[2115038364] 'agreement among raft nodes before linearized reading' (duration: 8.543287199s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:23.794148Z","caller":"traceutil/trace.go:172","msg":"trace[835124400] range","detail":"{range_begin:/registry/resourcequotas; range_end:; }","duration":"9.118371884s","start":"2026-03-26T14:51:14.675747Z","end":"2026-03-26T14:51:23.794119Z","steps":["trace[835124400] 'agreement among raft nodes before linearized reading' (duration: 9.118340854s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794593Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:11.469002Z","time spent":"12.325584795s","remote":"127.0.0.1:37308","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":0,"request content":"key:\"/registry/priorityclasses\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794647Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:15.250809Z","time spent":"8.5438326s","remote":"127.0.0.1:37146","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":0,"request content":"key:\"/registry/ipaddresses\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794676Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:14.675723Z","time spent":"9.118945635s","remote":"127.0.0.1:36688","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":0,"request content":"key:\"/registry/resourcequotas\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794180Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"10.041846718s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-26T14:51:23.794820Z","caller":"traceutil/trace.go:172","msg":"trace[296438971] range","detail":"{range_begin:/registry/validatingadmissionpolicybindings; range_end:; }","duration":"10.042485611s","start":"2026-03-26T14:51:13.752327Z","end":"2026-03-26T14:51:23.794812Z","steps":["trace[296438971] 'agreement among raft nodes before linearized reading' (duration: 10.041846548s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.794862Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:13.752267Z","time spent":"10.042585963s","remote":"127.0.0.1:37624","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.794127Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:10.806793Z","time spent":"12.987323509s","remote":"127.0.0.1:37724","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":0,"request content":"key:\"/registry/resourceslices\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:23.923450Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000240564s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:23.923535Z","caller":"traceutil/trace.go:172","msg":"trace[686935764] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000349266s","start":"2026-03-26T14:51:21.923168Z","end":"2026-03-26T14:51:23.923518Z","steps":["trace[686935764] 'agreement among raft nodes before linearized reading' (duration: 2.000237584s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:23.924249Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:21.923147Z","time spent":"2.001089931s","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:23 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:51:23.969224Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 653, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:24.294106Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486931,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:24.630425Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:17.630075Z","time spent":"7.00034344s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:24.795250Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486931,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-26T14:51:24.973292Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 654, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:25.247757Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000360101s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:25.247847Z","caller":"traceutil/trace.go:172","msg":"trace[996882503] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000458032s","start":"2026-03-26T14:51:23.247374Z","end":"2026-03-26T14:51:25.247832Z","steps":["trace[996882503] 'agreement among raft nodes before linearized reading' (duration: 2.00035909s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:25.248107Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.247336Z","time spent":"2.000641036s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:25 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:25.296159Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486931,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:25.796678Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486931,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:25.933285Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000230476s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:25.933376Z","caller":"traceutil/trace.go:172","msg":"trace[1376921556] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000360548s","start":"2026-03-26T14:51:23.932994Z","end":"2026-03-26T14:51:25.933355Z","steps":["trace[1376921556] 'agreement among raft nodes before linearized reading' (duration: 2.000225425s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:25.933430Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.932969Z","time spent":"2.00045205s","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:25 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-26T14:51:25.969216Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 654, vote: 5f170c75a1d97cc] ignored MsgPreVote from b59fd64328430f5b [logterm: 2, index: 641] at term 2: lease is not expired (remaining ticks: 1)"} {"level":"warn","ts":"2026-03-26T14:51:26.103780Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:19.103104Z","time spent":"7.000670617s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:26.297675Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486931,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:26.663904Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"16.695714882s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-26T14:51:26.664054Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"16.496306816s"} {"level":"warn","ts":"2026-03-26T14:51:26.664092Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486931,"received-request-id":10938290401367486915} {"level":"warn","ts":"2026-03-26T14:51:26.708947Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:15.169076Z","time spent":"11.539864599s","remote":"127.0.0.1:33194","response type":"/etcdserverpb.Maintenance/Status","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:26.709062Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"12.659580553s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:26.709142Z","caller":"traceutil/trace.go:172","msg":"trace[306957886] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:601; }","duration":"12.659674865s","start":"2026-03-26T14:51:14.049452Z","end":"2026-03-26T14:51:26.709127Z","steps":["trace[306957886] 'range keys from in-memory index tree' (duration: 12.65951823s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:26.709377Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.915063655s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:26.709390Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.567764645s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:26.709443Z","caller":"traceutil/trace.go:172","msg":"trace[287615090] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:601; }","duration":"2.915142326s","start":"2026-03-26T14:51:23.794286Z","end":"2026-03-26T14:51:26.709428Z","steps":["trace[287615090] 'range keys from in-memory index tree' (duration: 2.915047005s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:26.709456Z","caller":"traceutil/trace.go:172","msg":"trace[997113952] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:601; }","duration":"1.567881028s","start":"2026-03-26T14:51:25.141560Z","end":"2026-03-26T14:51:26.709441Z","steps":["trace[997113952] 'range keys from in-memory index tree' (duration: 1.567744785s)"],"step_count":1} {"level":"error","ts":"2026-03-26T14:51:26.709551Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[-]linearizable_read failed: etcdserver: request timed out\n[+]non_learner ok\n[+]data_corruption ok\n[+]serializable_read ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-26T14:51:26.798857Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486931,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:26.974525Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999422479s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:26.974605Z","caller":"traceutil/trace.go:172","msg":"trace[1523947521] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999518751s","start":"2026-03-26T14:51:21.975069Z","end":"2026-03-26T14:51:26.974588Z","steps":["trace[1523947521] 'agreement among raft nodes before linearized reading' (duration: 4.999418579s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:26.974658Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:21.975051Z","time spent":"4.999595432s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} 2026/03/26 14:51:26 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" 2026/03/26 14:51:26 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:27.107017Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"242.91313ms"} {"level":"info","ts":"2026-03-26T14:51:27.107094Z","caller":"traceutil/trace.go:172","msg":"trace[929674046] linearizableReadLoop","detail":"{readStateIndex:640; appliedIndex:641; }","duration":"3.313246149s","start":"2026-03-26T14:51:23.793818Z","end":"2026-03-26T14:51:27.107064Z","steps":["trace[929674046] 'read index received' (duration: 3.313226369s)","trace[929674046] 'applied index is now lower than readState.Index' (duration: 18.01µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.445358Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.740077427s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:27.445464Z","caller":"traceutil/trace.go:172","msg":"trace[1836017007] range","detail":"{range_begin:/registry/secrets; range_end:; response_count:0; response_revision:602; }","duration":"4.74019086s","start":"2026-03-26T14:51:22.705244Z","end":"2026-03-26T14:51:27.445435Z","steps":["trace[1836017007] 'agreement among raft nodes before linearized reading' (duration: 4.402167186s)","trace[1836017007] 'range keys from in-memory index tree' (duration: 337.871441ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.445509Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:22.705212Z","time spent":"4.740286191s","remote":"127.0.0.1:36718","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":27,"request content":"key:\"/registry/secrets\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.445751Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.110786925s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/roles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:27.445864Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.908331927s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/podtemplates\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:27.445881Z","caller":"traceutil/trace.go:172","msg":"trace[664647122] range","detail":"{range_begin:/registry/roles; range_end:; response_count:0; response_revision:602; }","duration":"7.110926398s","start":"2026-03-26T14:51:20.334916Z","end":"2026-03-26T14:51:27.445843Z","steps":["trace[664647122] 'agreement among raft nodes before linearized reading' (duration: 6.772529177s)","trace[664647122] 'range keys from in-memory index tree' (duration: 338.178487ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:27.445913Z","caller":"traceutil/trace.go:172","msg":"trace[1738765823] range","detail":"{range_begin:/registry/podtemplates; range_end:; response_count:0; response_revision:602; }","duration":"3.908376527s","start":"2026-03-26T14:51:23.537519Z","end":"2026-03-26T14:51:27.445895Z","steps":["trace[1738765823] 'agreement among raft nodes before linearized reading' (duration: 3.569776732s)","trace[1738765823] 'range keys from in-memory index tree' (duration: 338.519594ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.445940Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.537489Z","time spent":"3.908444988s","remote":"127.0.0.1:36780","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":27,"request content":"key:\"/registry/podtemplates\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.445911Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.467185818s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:27.445944Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:20.334900Z","time spent":"7.111029941s","remote":"127.0.0.1:37208","response type":"/etcdserverpb.KV/Range","request count":0,"request size":19,"response count":0,"response size":27,"request content":"key:\"/registry/roles\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.445995Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.529028188s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-1\" limit:1 ","response":"range_response_count:1 size:7523"} {"level":"warn","ts":"2026-03-26T14:51:27.445811Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.502768204s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:27.446055Z","caller":"traceutil/trace.go:172","msg":"trace[1967627669] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-1; range_end:; response_count:1; response_revision:602; }","duration":"7.529084749s","start":"2026-03-26T14:51:19.916949Z","end":"2026-03-26T14:51:27.446033Z","steps":["trace[1967627669] 'agreement among raft nodes before linearized reading' (duration: 7.190430882s)","trace[1967627669] 'range keys from in-memory index tree' (duration: 338.440592ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446078Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"10.624464756s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/jobs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:27.446088Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.779542603s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:27.446098Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:19.916927Z","time spent":"7.529158431s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":1,"response size":7545,"request content":"key:\"/registry/pods/kube-system/kube-apiserver-controller-1\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:27.446123Z","caller":"traceutil/trace.go:172","msg":"trace[162350184] range","detail":"{range_begin:/registry/jobs; range_end:; response_count:0; response_revision:602; }","duration":"10.624502826s","start":"2026-03-26T14:51:16.821604Z","end":"2026-03-26T14:51:27.446107Z","steps":["trace[162350184] 'agreement among raft nodes before linearized reading' (duration: 10.28583091s)","trace[162350184] 'range keys from in-memory index tree' (duration: 338.623936ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446161Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:16.821574Z","time spent":"10.624575768s","remote":"127.0.0.1:37004","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/jobs\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:27.446124Z","caller":"traceutil/trace.go:172","msg":"trace[1123044586] range","detail":"{range_begin:/registry/horizontalpodautoscalers; range_end:; response_count:0; response_revision:602; }","duration":"4.503062991s","start":"2026-03-26T14:51:22.943010Z","end":"2026-03-26T14:51:27.446073Z","steps":["trace[1123044586] 'agreement among raft nodes before linearized reading' (duration: 4.164359743s)","trace[1123044586] 'range keys from in-memory index tree' (duration: 338.178447ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446246Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.629359384s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/replicasets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:27.446041Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"10.598219883s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:27.446281Z","caller":"traceutil/trace.go:172","msg":"trace[463960160] range","detail":"{range_begin:/registry/replicasets; range_end:; response_count:0; response_revision:602; }","duration":"6.629396285s","start":"2026-03-26T14:51:20.816875Z","end":"2026-03-26T14:51:27.446271Z","steps":["trace[463960160] 'agreement among raft nodes before linearized reading' (duration: 6.290570355s)","trace[463960160] 'range keys from in-memory index tree' (duration: 338.709428ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:27.446295Z","caller":"traceutil/trace.go:172","msg":"trace[1617879678] range","detail":"{range_begin:/registry/services/endpoints; range_end:; response_count:0; response_revision:602; }","duration":"10.598470548s","start":"2026-03-26T14:51:16.847809Z","end":"2026-03-26T14:51:27.446280Z","steps":["trace[1617879678] 'agreement among raft nodes before linearized reading' (duration: 10.259623307s)","trace[1617879678] 'range keys from in-memory index tree' (duration: 338.584515ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446306Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:20.816852Z","time spent":"6.629447526s","remote":"127.0.0.1:37522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":27,"request content":"key:\"/registry/replicasets\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446327Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:16.847787Z","time spent":"10.598530478s","remote":"127.0.0.1:36858","response type":"/etcdserverpb.KV/Range","request count":0,"request size":32,"response count":0,"response size":27,"request content":"key:\"/registry/services/endpoints\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446332Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.317109879s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:27.446398Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.087695555s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:27.446428Z","caller":"traceutil/trace.go:172","msg":"trace[36735231] range","detail":"{range_begin:/registry/namespaces; range_end:; response_count:0; response_revision:602; }","duration":"8.317212201s","start":"2026-03-26T14:51:19.129193Z","end":"2026-03-26T14:51:27.446405Z","steps":["trace[36735231] 'agreement among raft nodes before linearized reading' (duration: 7.978204908s)","trace[36735231] 'range keys from in-memory index tree' (duration: 338.867251ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:27.446449Z","caller":"traceutil/trace.go:172","msg":"trace[1052274376] range","detail":"{range_begin:/registry/rolebindings; range_end:; response_count:0; response_revision:602; }","duration":"6.087744636s","start":"2026-03-26T14:51:21.358688Z","end":"2026-03-26T14:51:27.446432Z","steps":["trace[1052274376] 'agreement among raft nodes before linearized reading' (duration: 5.748707042s)","trace[1052274376] 'range keys from in-memory index tree' (duration: 338.966262ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446471Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:19.129178Z","time spent":"8.317283862s","remote":"127.0.0.1:36736","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":27,"request content":"key:\"/registry/namespaces\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446489Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:21.358665Z","time spent":"6.087812956s","remote":"127.0.0.1:37242","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":27,"request content":"key:\"/registry/rolebindings\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446491Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"9.571580902s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.157\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-26T14:51:27.446159Z","caller":"traceutil/trace.go:172","msg":"trace[241842740] range","detail":"{range_begin:/registry/controllers; range_end:; response_count:0; response_revision:602; }","duration":"4.467254409s","start":"2026-03-26T14:51:22.978718Z","end":"2026-03-26T14:51:27.445972Z","steps":["trace[241842740] 'agreement among raft nodes before linearized reading' (duration: 4.128641543s)","trace[241842740] 'range keys from in-memory index tree' (duration: 338.529374ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446522Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.383625838s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets/kube-system/kubeadm-certs\" limit:1 ","response":"range_response_count:1 size:11447"} {"level":"info","ts":"2026-03-26T14:51:27.446537Z","caller":"traceutil/trace.go:172","msg":"trace[1943595633] range","detail":"{range_begin:/registry/masterleases/199.204.45.157; range_end:; response_count:1; response_revision:602; }","duration":"9.571626012s","start":"2026-03-26T14:51:17.874897Z","end":"2026-03-26T14:51:27.446523Z","steps":["trace[1943595633] 'agreement among raft nodes before linearized reading' (duration: 9.232542328s)","trace[1943595633] 'range keys from in-memory index tree' (duration: 338.945542ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446499Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"9.488310318s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/poddisruptionbudgets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:27.446557Z","caller":"traceutil/trace.go:172","msg":"trace[20839982] range","detail":"{range_begin:/registry/secrets/kube-system/kubeadm-certs; range_end:; response_count:1; response_revision:602; }","duration":"7.383658219s","start":"2026-03-26T14:51:20.062888Z","end":"2026-03-26T14:51:27.446546Z","steps":["trace[20839982] 'agreement among raft nodes before linearized reading' (duration: 7.044507973s)","trace[20839982] 'range keys from in-memory index tree' (duration: 338.982273ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446556Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:22.978701Z","time spent":"4.46783943s","remote":"127.0.0.1:36964","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":27,"request content":"key:\"/registry/controllers\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446575Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:17.874869Z","time spent":"9.571695534s","remote":"127.0.0.1:36606","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":157,"request content":"key:\"/registry/masterleases/199.204.45.157\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:27.446592Z","caller":"traceutil/trace.go:172","msg":"trace[1353154753] range","detail":"{range_begin:/registry/poddisruptionbudgets; range_end:; response_count:0; response_revision:602; }","duration":"9.488399939s","start":"2026-03-26T14:51:17.958172Z","end":"2026-03-26T14:51:27.446572Z","steps":["trace[1353154753] 'agreement among raft nodes before linearized reading' (duration: 9.149254083s)","trace[1353154753] 'range keys from in-memory index tree' (duration: 338.405682ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446607Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.601244288s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumeclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:27.446631Z","caller":"traceutil/trace.go:172","msg":"trace[915458271] range","detail":"{range_begin:/registry/persistentvolumeclaims; range_end:; response_count:0; response_revision:602; }","duration":"6.601265068s","start":"2026-03-26T14:51:20.845357Z","end":"2026-03-26T14:51:27.446623Z","steps":["trace[915458271] 'agreement among raft nodes before linearized reading' (duration: 6.262158323s)","trace[915458271] 'range keys from in-memory index tree' (duration: 339.073485ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446655Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:20.845334Z","time spent":"6.60131457s","remote":"127.0.0.1:36848","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":27,"request content":"key:\"/registry/persistentvolumeclaims\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446659Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:17.958148Z","time spent":"9.488500271s","remote":"127.0.0.1:37198","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":27,"request content":"key:\"/registry/poddisruptionbudgets\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:27.446699Z","caller":"traceutil/trace.go:172","msg":"trace[1236907779] range","detail":"{range_begin:/registry/leases; range_end:; response_count:0; response_revision:602; }","duration":"4.779581354s","start":"2026-03-26T14:51:22.666535Z","end":"2026-03-26T14:51:27.446117Z","steps":["trace[1236907779] 'agreement among raft nodes before linearized reading' (duration: 4.440883006s)","trace[1236907779] 'range keys from in-memory index tree' (duration: 338.074495ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446726Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.154483095s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/prioritylevelconfigurations\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:27.446240Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:22.942990Z","time spent":"4.503216633s","remote":"127.0.0.1:36984","response type":"/etcdserverpb.KV/Range","request count":0,"request size":38,"response count":0,"response size":27,"request content":"key:\"/registry/horizontalpodautoscalers\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:27.446766Z","caller":"traceutil/trace.go:172","msg":"trace[1224252925] range","detail":"{range_begin:/registry/prioritylevelconfigurations; range_end:; response_count:0; response_revision:602; }","duration":"8.154516966s","start":"2026-03-26T14:51:19.292234Z","end":"2026-03-26T14:51:27.446751Z","steps":["trace[1224252925] 'agreement among raft nodes before linearized reading' (duration: 7.815154266s)","trace[1224252925] 'range keys from in-memory index tree' (duration: 339.317519ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.446792Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:19.292212Z","time spent":"8.154573196s","remote":"127.0.0.1:37454","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":0,"response size":27,"request content":"key:\"/registry/prioritylevelconfigurations\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446582Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:20.062866Z","time spent":"7.38370965s","remote":"127.0.0.1:36718","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":11469,"request content":"key:\"/registry/secrets/kube-system/kubeadm-certs\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446732Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:22.666514Z","time spent":"4.780209885s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":20,"response count":0,"response size":27,"request content":"key:\"/registry/leases\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.446955Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.362487133s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:45 size:3836"} {"level":"info","ts":"2026-03-26T14:51:27.447008Z","caller":"traceutil/trace.go:172","msg":"trace[499576965] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:45; response_revision:602; }","duration":"8.362539215s","start":"2026-03-26T14:51:19.084453Z","end":"2026-03-26T14:51:27.446992Z","steps":["trace[499576965] 'agreement among raft nodes before linearized reading' (duration: 8.022954481s)","trace[499576965] 'range keys from in-memory index tree' (duration: 339.285829ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.447054Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:19.084422Z","time spent":"8.362618486s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":45,"response size":3858,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"warn","ts":"2026-03-26T14:51:27.447300Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"5.913313563s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:27.447310Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.629835027s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattachments\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:27.447335Z","caller":"traceutil/trace.go:172","msg":"trace[35787340] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices; range_end:; response_count:0; response_revision:602; }","duration":"5.913348244s","start":"2026-03-26T14:51:21.533976Z","end":"2026-03-26T14:51:27.447324Z","steps":["trace[35787340] 'agreement among raft nodes before linearized reading' (duration: 5.573449443s)","trace[35787340] 'range keys from in-memory index tree' (duration: 339.85332ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:27.447352Z","caller":"traceutil/trace.go:172","msg":"trace[1335503701] range","detail":"{range_begin:/registry/volumeattachments; range_end:; response_count:0; response_revision:602; }","duration":"8.629878298s","start":"2026-03-26T14:51:18.817462Z","end":"2026-03-26T14:51:27.447340Z","steps":["trace[1335503701] 'agreement among raft nodes before linearized reading' (duration: 8.289954667s)","trace[1335503701] 'range keys from in-memory index tree' (duration: 339.86348ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:27.447367Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:21.533948Z","time spent":"5.913409694s","remote":"127.0.0.1:37736","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":27,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.447380Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:18.817439Z","time spent":"8.629934019s","remote":"127.0.0.1:37348","response type":"/etcdserverpb.KV/Range","request count":0,"request size":31,"response count":0,"response size":27,"request content":"key:\"/registry/volumeattachments\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:27.447651Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"340.107404ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:51:27.448928Z","caller":"traceutil/trace.go:172","msg":"trace[1718106380] transaction","detail":"{read_only:false; number_of_response:1; response_revision:611; }","duration":"1.340711785s","start":"2026-03-26T14:51:26.108197Z","end":"2026-03-26T14:51:27.448909Z","steps":["trace[1718106380] 'process raft request' (duration: 1.340673544s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:27.449089Z","caller":"traceutil/trace.go:172","msg":"trace[1000129926] transaction","detail":"{read_only:false; response_revision:611; number_of_response:1; }","duration":"6.447650962s","start":"2026-03-26T14:51:21.001421Z","end":"2026-03-26T14:51:27.449072Z","steps":["trace[1000129926] 'process raft request' (duration: 6.447327826s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:27.449152Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:26.108162Z","time spent":"1.340882298s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":49,"response count":0,"response size":584,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:27.449221Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.342341Z","time spent":"4.106877205s","remote":"127.0.0.1:37642","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-26T14:51:27.449291Z","caller":"traceutil/trace.go:172","msg":"trace[1963401421] transaction","detail":"{read_only:false; number_of_response:1; response_revision:611; }","duration":"2.814664021s","start":"2026-03-26T14:51:24.634619Z","end":"2026-03-26T14:51:27.449283Z","steps":["trace[1963401421] 'process raft request' (duration: 2.814206902s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:27.449341Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:24.634588Z","time spent":"2.814725053s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":69,"response count":0,"response size":724,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:27.449240Z","caller":"traceutil/trace.go:172","msg":"trace[313460947] transaction","detail":"{read_only:false; response_revision:609; number_of_response:1; }","duration":"6.93575718s","start":"2026-03-26T14:51:20.513470Z","end":"2026-03-26T14:51:27.449227Z","steps":["trace[313460947] 'process raft request' (duration: 6.935182679s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:27.449205Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:21.001387Z","time spent":"6.447752124s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":411,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:27.449420Z","caller":"traceutil/trace.go:172","msg":"trace[1087141173] transaction","detail":"{read_only:false; response_revision:610; number_of_response:1; }","duration":"6.545395203s","start":"2026-03-26T14:51:20.903869Z","end":"2026-03-26T14:51:27.449264Z","steps":["trace[1087141173] 'process raft request' (duration: 6.544841412s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:27.449858Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:20.903832Z","time spent":"6.545907313s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":778,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-26T14:51:27.449461Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:20.513443Z","time spent":"6.935994285s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":415,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:27.607246Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486935,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:27.942522Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000792908s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:27.942608Z","caller":"traceutil/trace.go:172","msg":"trace[1270747232] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.00089401s","start":"2026-03-26T14:51:25.941695Z","end":"2026-03-26T14:51:27.942589Z","steps":["trace[1270747232] 'agreement among raft nodes before linearized reading' (duration: 2.000789128s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:27.942667Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:25.941676Z","time spent":"2.000980051s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:27 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:28.108395Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486935,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:28.217430Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.11019779s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-26T14:51:28.217550Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"910.446026ms"} {"level":"warn","ts":"2026-03-26T14:51:28.217667Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486935,"received-request-id":10938290401367486931} {"level":"warn","ts":"2026-03-26T14:51:28.609155Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486935,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:29.109713Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486935,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:29.314679Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.096820297s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-26T14:51:29.372165Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.105659109s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-26T14:51:29.372319Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"954.743085ms"} {"level":"info","ts":"2026-03-26T14:51:29.372460Z","caller":"traceutil/trace.go:172","msg":"trace[103471844] linearizableReadLoop","detail":"{readStateIndex:655; appliedIndex:660; }","duration":"2.265288562s","start":"2026-03-26T14:51:27.107148Z","end":"2026-03-26T14:51:29.372437Z","steps":["trace[103471844] 'read index received' (duration: 2.265272461s)","trace[103471844] 'applied index is now lower than readState.Index' (duration: 14.601µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.372543Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"5.551541621s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourcequotas\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.372633Z","caller":"traceutil/trace.go:172","msg":"trace[1964562626] range","detail":"{range_begin:/registry/resourcequotas; range_end:; response_count:0; response_revision:614; }","duration":"5.551644603s","start":"2026-03-26T14:51:23.820972Z","end":"2026-03-26T14:51:29.372617Z","steps":["trace[1964562626] 'agreement among raft nodes before linearized reading' (duration: 5.55149041s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:29.372671Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.820969Z","time spent":"5.551693924s","remote":"127.0.0.1:36688","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/resourcequotas\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.860798Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.172058103s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingressclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.860822Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"488.418681ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"warn","ts":"2026-03-26T14:51:29.860843Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.039865571s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingwebhookconfigurations\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.860863Z","caller":"traceutil/trace.go:172","msg":"trace[2044882392] range","detail":"{range_begin:/registry/ingressclasses; range_end:; response_count:0; response_revision:615; }","duration":"3.172113844s","start":"2026-03-26T14:51:26.688730Z","end":"2026-03-26T14:51:29.860844Z","steps":["trace[2044882392] 'agreement among raft nodes before linearized reading' (duration: 2.683865156s)","trace[2044882392] 'range keys from in-memory index tree' (duration: 488.179337ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:29.860872Z","caller":"traceutil/trace.go:172","msg":"trace[379848771] range","detail":"{range_begin:/registry/validatingwebhookconfigurations; range_end:; response_count:0; response_revision:615; }","duration":"6.039889311s","start":"2026-03-26T14:51:23.820971Z","end":"2026-03-26T14:51:29.860860Z","steps":["trace[379848771] 'agreement among raft nodes before linearized reading' (duration: 5.551651303s)","trace[379848771] 'range keys from in-memory index tree' (duration: 488.203598ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.860864Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.040056136s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaimtemplates\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.860900Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:26.688719Z","time spent":"3.172172476s","remote":"127.0.0.1:37142","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/ingressclasses\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:29.860916Z","caller":"traceutil/trace.go:172","msg":"trace[22615153] range","detail":"{range_begin:/registry/resourceclaimtemplates; range_end:; response_count:0; response_revision:615; }","duration":"6.040120867s","start":"2026-03-26T14:51:23.820783Z","end":"2026-03-26T14:51:29.860904Z","steps":["trace[22615153] 'agreement among raft nodes before linearized reading' (duration: 5.551733236s)","trace[22615153] 'range keys from in-memory index tree' (duration: 488.107685ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.860932Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.041376341s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.860956Z","caller":"traceutil/trace.go:172","msg":"trace[1397446050] range","detail":"{range_begin:/registry/validatingadmissionpolicybindings; range_end:; response_count:0; response_revision:615; }","duration":"6.041398172s","start":"2026-03-26T14:51:23.819549Z","end":"2026-03-26T14:51:29.860948Z","steps":["trace[1397446050] 'agreement among raft nodes before linearized reading' (duration: 5.553078732s)","trace[1397446050] 'range keys from in-memory index tree' (duration: 488.288528ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.860953Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.820770Z","time spent":"6.040172568s","remote":"127.0.0.1:37684","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":27,"request content":"key:\"/registry/resourceclaimtemplates\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.860978Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.819531Z","time spent":"6.041440892s","remote":"127.0.0.1:37624","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":27,"request content":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861035Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.056970153s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.861059Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.641219487s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/runtimeclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.861080Z","caller":"traceutil/trace.go:172","msg":"trace[1293236269] range","detail":"{range_begin:/registry/resourceslices; range_end:; response_count:0; response_revision:615; }","duration":"6.057022964s","start":"2026-03-26T14:51:23.804045Z","end":"2026-03-26T14:51:29.861068Z","steps":["trace[1293236269] 'agreement among raft nodes before linearized reading' (duration: 5.568612803s)","trace[1293236269] 'range keys from in-memory index tree' (duration: 488.3408ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:29.861084Z","caller":"traceutil/trace.go:172","msg":"trace[74004526] range","detail":"{range_begin:/registry/runtimeclasses; range_end:; response_count:0; response_revision:615; }","duration":"4.641242827s","start":"2026-03-26T14:51:25.219833Z","end":"2026-03-26T14:51:29.861075Z","steps":["trace[74004526] 'agreement among raft nodes before linearized reading' (duration: 4.152801806s)","trace[74004526] 'range keys from in-memory index tree' (duration: 488.407401ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861103Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:25.219807Z","time spent":"4.641291129s","remote":"127.0.0.1:37174","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/runtimeclasses\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861103Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.804028Z","time spent":"6.057069975s","remote":"127.0.0.1:37724","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/resourceslices\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861120Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.038004034s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllerrevisions\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.861130Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.151626452s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.861144Z","caller":"traceutil/trace.go:172","msg":"trace[1521922073] range","detail":"{range_begin:/registry/controllerrevisions; range_end:; response_count:0; response_revision:615; }","duration":"6.038027574s","start":"2026-03-26T14:51:23.823109Z","end":"2026-03-26T14:51:29.861136Z","steps":["trace[1521922073] 'agreement among raft nodes before linearized reading' (duration: 5.549552482s)","trace[1521922073] 'range keys from in-memory index tree' (duration: 488.441282ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:29.861156Z","caller":"traceutil/trace.go:172","msg":"trace[1173801967] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:615; }","duration":"3.151648702s","start":"2026-03-26T14:51:26.709497Z","end":"2026-03-26T14:51:29.861146Z","steps":["trace[1173801967] 'agreement among raft nodes before linearized reading' (duration: 2.663084549s)","trace[1173801967] 'range keys from in-memory index tree' (duration: 488.532943ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861166Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.823100Z","time spent":"6.038060475s","remote":"127.0.0.1:37544","response type":"/etcdserverpb.KV/Range","request count":0,"request size":33,"response count":0,"response size":27,"request content":"key:\"/registry/controllerrevisions\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861167Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.041754979s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.861193Z","caller":"traceutil/trace.go:172","msg":"trace[2067592826] range","detail":"{range_begin:/registry/resourceslices; range_end:; response_count:0; response_revision:615; }","duration":"6.041780849s","start":"2026-03-26T14:51:23.819404Z","end":"2026-03-26T14:51:29.861185Z","steps":["trace[2067592826] 'agreement among raft nodes before linearized reading' (duration: 5.553245166s)","trace[2067592826] 'range keys from in-memory index tree' (duration: 488.499563ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861192Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.041731198s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ipaddresses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.861212Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.819386Z","time spent":"6.04182098s","remote":"127.0.0.1:37724","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/resourceslices\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.860789Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.037588906s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingress\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.861233Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.038075965s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.861239Z","caller":"traceutil/trace.go:172","msg":"trace[1139187053] range","detail":"{range_begin:/registry/ipaddresses; range_end:; response_count:0; response_revision:615; }","duration":"6.041761049s","start":"2026-03-26T14:51:23.819455Z","end":"2026-03-26T14:51:29.861216Z","steps":["trace[1139187053] 'agreement among raft nodes before linearized reading' (duration: 5.553182085s)","trace[1139187053] 'range keys from in-memory index tree' (duration: 488.539343ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:29.861248Z","caller":"traceutil/trace.go:172","msg":"trace[338646347] range","detail":"{range_begin:/registry/ingress; range_end:; response_count:0; response_revision:615; }","duration":"6.038059875s","start":"2026-03-26T14:51:23.823178Z","end":"2026-03-26T14:51:29.861237Z","steps":["trace[338646347] 'agreement among raft nodes before linearized reading' (duration: 5.549468671s)","trace[338646347] 'range keys from in-memory index tree' (duration: 488.098655ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861273Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.040390051s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceclaims\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.861277Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.819448Z","time spent":"6.04182014s","remote":"127.0.0.1:37146","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":27,"request content":"key:\"/registry/ipaddresses\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861291Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.04034851s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.861289Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.823167Z","time spent":"6.038112226s","remote":"127.0.0.1:37116","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":27,"request content":"key:\"/registry/ingress\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:29.861306Z","caller":"traceutil/trace.go:172","msg":"trace[1226936911] range","detail":"{range_begin:/registry/rolebindings; range_end:; response_count:0; response_revision:615; }","duration":"6.04036926s","start":"2026-03-26T14:51:23.820932Z","end":"2026-03-26T14:51:29.861302Z","steps":["trace[1226936911] 'agreement among raft nodes before linearized reading' (duration: 5.551603753s)","trace[1226936911] 'range keys from in-memory index tree' (duration: 488.720177ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.860906Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.820964Z","time spent":"6.039927472s","remote":"127.0.0.1:37556","response type":"/etcdserverpb.KV/Range","request count":0,"request size":45,"response count":0,"response size":27,"request content":"key:\"/registry/validatingwebhookconfigurations\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861316Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.040446782s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.861317Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.820925Z","time spent":"6.040389031s","remote":"127.0.0.1:37242","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":27,"request content":"key:\"/registry/rolebindings\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:29.861333Z","caller":"traceutil/trace.go:172","msg":"trace[449878888] range","detail":"{range_begin:/registry/secrets; range_end:; response_count:0; response_revision:615; }","duration":"6.040462252s","start":"2026-03-26T14:51:23.820865Z","end":"2026-03-26T14:51:29.861327Z","steps":["trace[449878888] 'agreement among raft nodes before linearized reading' (duration: 5.551806206s)","trace[449878888] 'range keys from in-memory index tree' (duration: 488.632206ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861348Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.820853Z","time spent":"6.040490272s","remote":"127.0.0.1:36718","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":27,"request content":"key:\"/registry/secrets\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861096Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.037899592s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterroles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.861295Z","caller":"traceutil/trace.go:172","msg":"trace[1584772263] range","detail":"{range_begin:/registry/resourceclaims; range_end:; response_count:0; response_revision:615; }","duration":"6.040412731s","start":"2026-03-26T14:51:23.820875Z","end":"2026-03-26T14:51:29.861288Z","steps":["trace[1584772263] 'agreement among raft nodes before linearized reading' (duration: 5.551794516s)","trace[1584772263] 'range keys from in-memory index tree' (duration: 488.585525ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861136Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.037936093s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/priorityclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:29.861457Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.820856Z","time spent":"6.040595914s","remote":"127.0.0.1:37676","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/resourceclaims\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:29.861250Z","caller":"traceutil/trace.go:172","msg":"trace[2072815373] range","detail":"{range_begin:/registry/resourceclaims; range_end:; response_count:0; response_revision:615; }","duration":"6.038092625s","start":"2026-03-26T14:51:23.823152Z","end":"2026-03-26T14:51:29.861245Z","steps":["trace[2072815373] 'agreement among raft nodes before linearized reading' (duration: 5.549503781s)","trace[2072815373] 'range keys from in-memory index tree' (duration: 488.562834ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:29.861471Z","caller":"traceutil/trace.go:172","msg":"trace[816823184] range","detail":"{range_begin:/registry/priorityclasses; range_end:; response_count:0; response_revision:615; }","duration":"6.038286079s","start":"2026-03-26T14:51:23.823176Z","end":"2026-03-26T14:51:29.861463Z","steps":["trace[816823184] 'agreement among raft nodes before linearized reading' (duration: 5.549465791s)","trace[816823184] 'range keys from in-memory index tree' (duration: 488.448122ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861484Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.823149Z","time spent":"6.03832603s","remote":"127.0.0.1:37676","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/resourceclaims\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861501Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.823162Z","time spent":"6.03832831s","remote":"127.0.0.1:37308","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":27,"request content":"key:\"/registry/priorityclasses\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861197Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.183939495s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.861526Z","caller":"traceutil/trace.go:172","msg":"trace[1239577777] range","detail":"{range_begin:/registry/leases; range_end:; response_count:0; response_revision:615; }","duration":"3.184268321s","start":"2026-03-26T14:51:26.677252Z","end":"2026-03-26T14:51:29.861520Z","steps":["trace[1239577777] 'agreement among raft nodes before linearized reading' (duration: 2.695376581s)","trace[1239577777] 'range keys from in-memory index tree' (duration: 488.555353ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861544Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:26.677233Z","time spent":"3.184306472s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":20,"response count":0,"response size":27,"request content":"key:\"/registry/leases\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:29.861450Z","caller":"traceutil/trace.go:172","msg":"trace[247000882] range","detail":"{range_begin:/registry/clusterroles; range_end:; response_count:0; response_revision:615; }","duration":"6.038252109s","start":"2026-03-26T14:51:23.823188Z","end":"2026-03-26T14:51:29.861440Z","steps":["trace[247000882] 'agreement among raft nodes before linearized reading' (duration: 5.549375359s)","trace[247000882] 'range keys from in-memory index tree' (duration: 488.505223ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861608Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.823184Z","time spent":"6.038421142s","remote":"127.0.0.1:37260","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":27,"request content":"key:\"/registry/clusterroles\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.861419Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"6.040659786s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:29.861639Z","caller":"traceutil/trace.go:172","msg":"trace[1201778275] range","detail":"{range_begin:/registry/services/specs; range_end:; response_count:0; response_revision:615; }","duration":"6.040878601s","start":"2026-03-26T14:51:23.820751Z","end":"2026-03-26T14:51:29.861630Z","steps":["trace[1201778275] 'agreement among raft nodes before linearized reading' (duration: 5.551815797s)","trace[1201778275] 'range keys from in-memory index tree' (duration: 488.825529ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:29.861685Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:23.820735Z","time spent":"6.040924542s","remote":"127.0.0.1:36924","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/services/specs\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:29.873097Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486943,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:29.943059Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000141596s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-26T14:51:29.943204Z","caller":"traceutil/trace.go:172","msg":"trace[1553021377] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000291179s","start":"2026-03-26T14:51:27.942899Z","end":"2026-03-26T14:51:29.943190Z","steps":["trace[1553021377] 'agreement among raft nodes before linearized reading' (duration: 2.000140876s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:29.943300Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:27.942871Z","time spent":"2.000419641s","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:29 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:30.234321Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"661.971591ms"} {"level":"warn","ts":"2026-03-26T14:51:30.234407Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486943,"received-request-id":10938290401367486935} {"level":"warn","ts":"2026-03-26T14:51:30.271149Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"410.253747ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"warn","ts":"2026-03-26T14:51:30.866790Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"595.52593ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"warn","ts":"2026-03-26T14:51:30.867303Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:27.449941Z","time spent":"3.417357666s","remote":"127.0.0.1:36606","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-26T14:51:30.868041Z","caller":"traceutil/trace.go:172","msg":"trace[1022257512] transaction","detail":"{read_only:false; response_revision:617; number_of_response:1; }","duration":"3.416192703s","start":"2026-03-26T14:51:27.451817Z","end":"2026-03-26T14:51:30.868010Z","steps":["trace[1022257512] 'process raft request' (duration: 3.415382638s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.868243Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:27.451789Z","time spent":"3.416359325s","remote":"127.0.0.1:37642","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":534,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-26T14:51:30.868510Z","caller":"traceutil/trace.go:172","msg":"trace[1885786167] linearizableReadLoop","detail":"{readStateIndex:667; appliedIndex:661; }","duration":"1.495963812s","start":"2026-03-26T14:51:29.372524Z","end":"2026-03-26T14:51:30.868488Z","steps":["trace[1885786167] 'read index received' (duration: 862.9657ms)","trace[1885786167] 'applied index is now lower than readState.Index' (duration: 632.996812ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:30.868973Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.413151013s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-26T14:51:30.868523Z","caller":"traceutil/trace.go:172","msg":"trace[324059984] transaction","detail":"{read_only:false; response_revision:618; number_of_response:1; }","duration":"3.412334648s","start":"2026-03-26T14:51:27.456160Z","end":"2026-03-26T14:51:30.868495Z","steps":["trace[324059984] 'process raft request' (duration: 3.411725747s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:30.869033Z","caller":"traceutil/trace.go:172","msg":"trace[1809681663] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649; range_end:; response_count:1; response_revision:619; }","duration":"3.413224696s","start":"2026-03-26T14:51:27.455794Z","end":"2026-03-26T14:51:30.869019Z","steps":["trace[1809681663] 'agreement among raft nodes before linearized reading' (duration: 3.413042311s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.869077Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:27.455784Z","time spent":"3.413280767s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":816,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:30.869115Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:27.456105Z","time spent":"3.41294966s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7700,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:30.869407Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.414540051s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-node-lease/controller-1\" limit:1 ","response":"range_response_count:1 size:551"} {"level":"info","ts":"2026-03-26T14:51:30.869452Z","caller":"traceutil/trace.go:172","msg":"trace[25335621] range","detail":"{range_begin:/registry/leases/kube-node-lease/controller-1; range_end:; response_count:1; response_revision:622; }","duration":"3.414585551s","start":"2026-03-26T14:51:27.454855Z","end":"2026-03-26T14:51:30.869441Z","steps":["trace[25335621] 'agreement among raft nodes before linearized reading' (duration: 3.414415468s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.869481Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:27.454843Z","time spent":"3.414632182s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":49,"response count":1,"response size":573,"request content":"key:\"/registry/leases/kube-node-lease/controller-1\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:30.869619Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.414880747s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/apiserver-tyzveoctxbnh6lvbkdt2xqncle\" limit:1 ","response":"range_response_count:1 size:691"} {"level":"info","ts":"2026-03-26T14:51:30.869656Z","caller":"traceutil/trace.go:172","msg":"trace[808779404] range","detail":"{range_begin:/registry/leases/kube-system/apiserver-tyzveoctxbnh6lvbkdt2xqncle; range_end:; response_count:1; response_revision:622; }","duration":"3.414911677s","start":"2026-03-26T14:51:27.454732Z","end":"2026-03-26T14:51:30.869644Z","steps":["trace[808779404] 'agreement among raft nodes before linearized reading' (duration: 3.414829306s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:30.869681Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:27.454718Z","time spent":"3.414957008s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":69,"response count":1,"response size":713,"request content":"key:\"/registry/leases/kube-system/apiserver-tyzveoctxbnh6lvbkdt2xqncle\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:31.043457Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"607.78939ms"} {"level":"warn","ts":"2026-03-26T14:51:31.368799Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486944,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:31.856509Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"812.765411ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:5"} {"level":"warn","ts":"2026-03-26T14:51:31.870063Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486944,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:31.953092Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999879644s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-26T14:51:31.953188Z","caller":"traceutil/trace.go:172","msg":"trace[1963942308] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.999990457s","start":"2026-03-26T14:51:29.953174Z","end":"2026-03-26T14:51:31.953165Z","steps":["trace[1963942308] 'agreement among raft nodes before linearized reading' (duration: 1.999875443s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:31.953247Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:29.953152Z","time spent":"2.000086368s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/26 14:51:31 WARNING: [core] [Server #5]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-26T14:51:31.977792Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"734.221963ms"} {"level":"info","ts":"2026-03-26T14:51:31.977869Z","caller":"traceutil/trace.go:172","msg":"trace[323253717] linearizableReadLoop","detail":"{readStateIndex:671; appliedIndex:672; }","duration":"1.109262316s","start":"2026-03-26T14:51:30.868573Z","end":"2026-03-26T14:51:31.977835Z","steps":["trace[323253717] 'read index received' (duration: 1.109252795s)","trace[323253717] 'applied index is now lower than readState.Index' (duration: 8.07µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:31.978107Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.154889675s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:31.978108Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.115396349s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:31.978155Z","caller":"traceutil/trace.go:172","msg":"trace[1805893697] range","detail":"{range_begin:/registry/pods; range_end:; response_count:0; response_revision:622; }","duration":"2.154961706s","start":"2026-03-26T14:51:29.823181Z","end":"2026-03-26T14:51:31.978143Z","steps":["trace[1805893697] 'agreement among raft nodes before linearized reading' (duration: 2.154838054s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:31.978185Z","caller":"traceutil/trace.go:172","msg":"trace[803907695] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:622; }","duration":"2.115478671s","start":"2026-03-26T14:51:29.862683Z","end":"2026-03-26T14:51:31.978161Z","steps":["trace[803907695] 'agreement among raft nodes before linearized reading' (duration: 2.115350828s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:31.978200Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:29.823165Z","time spent":"2.155026367s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/pods\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:31.978520Z","caller":"txn/util.go:46","msg":"failed to apply request","took":"36.351µs","request":"header: lease_revoke:","response":"size:27","error":"lease not found"} {"level":"warn","ts":"2026-03-26T14:51:31.978618Z","caller":"etcdserver/server.go:900","msg":"failed to revoke lease","lease-id":"17cc9d2a9f92f145","error":"lease not found"} {"level":"info","ts":"2026-03-26T14:51:31.978722Z","caller":"traceutil/trace.go:172","msg":"trace[1543728228] transaction","detail":"{read_only:false; number_of_response:1; response_revision:625; }","duration":"1.109719145s","start":"2026-03-26T14:51:30.868955Z","end":"2026-03-26T14:51:31.978674Z","steps":["trace[1543728228] 'process raft request' (duration: 1.109624253s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:31.978751Z","caller":"traceutil/trace.go:172","msg":"trace[1070443179] transaction","detail":"{read_only:false; response_revision:628; number_of_response:1; }","duration":"1.102827623s","start":"2026-03-26T14:51:30.875910Z","end":"2026-03-26T14:51:31.978737Z","steps":["trace[1070443179] 'process raft request' (duration: 1.102781192s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:31.978788Z","caller":"traceutil/trace.go:172","msg":"trace[778362543] transaction","detail":"{read_only:false; response_revision:626; number_of_response:1; }","duration":"1.106179677s","start":"2026-03-26T14:51:30.872548Z","end":"2026-03-26T14:51:31.978727Z","steps":["trace[778362543] 'process raft request' (duration: 1.106058285s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:31.978819Z","caller":"traceutil/trace.go:172","msg":"trace[609044452] transaction","detail":"{read_only:false; response_revision:627; number_of_response:1; }","duration":"1.104597178s","start":"2026-03-26T14:51:30.874152Z","end":"2026-03-26T14:51:31.978749Z","steps":["trace[609044452] 'process raft request' (duration: 1.104507626s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:31.978845Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.875890Z","time spent":"1.102890365s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:31.978887Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.872518Z","time spent":"1.106315979s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:31.978880Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.868930Z","time spent":"1.109845407s","remote":"127.0.0.1:36606","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":41,"response count":0,"response size":36,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:31.979003Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.874130Z","time spent":"1.104716149s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:32.071197Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486950,"received-request-id":10938290401367486944} {"level":"info","ts":"2026-03-26T14:51:32.239193Z","caller":"traceutil/trace.go:172","msg":"trace[1052652500] linearizableReadLoop","detail":"{readStateIndex:680; appliedIndex:680; }","duration":"261.19268ms","start":"2026-03-26T14:51:31.977969Z","end":"2026-03-26T14:51:32.239162Z","steps":["trace[1052652500] 'read index received' (duration: 261.18146ms)","trace[1052652500] 'applied index is now lower than readState.Index' (duration: 9.46µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:32.536339Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"97.077658ms"} {"level":"warn","ts":"2026-03-26T14:51:32.739859Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486953,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:33.240618Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486953,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:33.741509Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486953,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:33.750602Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.796946741s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:33.750721Z","caller":"traceutil/trace.go:172","msg":"trace[1431465899] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:628; }","duration":"1.797075464s","start":"2026-03-26T14:51:31.953617Z","end":"2026-03-26T14:51:33.750692Z","steps":["trace[1431465899] 'agreement among raft nodes before linearized reading' (duration: 285.646651ms)","trace[1431465899] 'range keys from in-memory index tree' (duration: 1.51124418s)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:33.750722Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.679384991s","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:39"} {"level":"warn","ts":"2026-03-26T14:51:33.750773Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:31.953577Z","time spent":"1.797185346s","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:33.750940Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:31.980890Z","time spent":"1.770046434s","remote":"127.0.0.1:36606","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:33.751204Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.877107927s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-vip-controller-1\" limit:1 ","response":"range_response_count:1 size:4857"} {"level":"info","ts":"2026-03-26T14:51:33.751309Z","caller":"traceutil/trace.go:172","msg":"trace[748228857] range","detail":"{range_begin:/registry/pods/kube-system/kube-vip-controller-1; range_end:; response_count:1; response_revision:628; }","duration":"2.877223889s","start":"2026-03-26T14:51:30.874035Z","end":"2026-03-26T14:51:33.751258Z","steps":["trace[748228857] 'agreement among raft nodes before linearized reading' (duration: 1.365281317s)","trace[748228857] 'range keys from in-memory index tree' (duration: 1.511642387s)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:33.751386Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:30.874025Z","time spent":"2.877317831s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Range","request count":0,"request size":52,"response count":1,"response size":4879,"request content":"key:\"/registry/pods/kube-system/kube-vip-controller-1\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:33.831802Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.295183619s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-26T14:51:33.831926Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.095455215s"} {"level":"info","ts":"2026-03-26T14:51:33.832058Z","caller":"traceutil/trace.go:172","msg":"trace[797805650] linearizableReadLoop","detail":"{readStateIndex:684; appliedIndex:685; }","duration":"1.592758287s","start":"2026-03-26T14:51:32.239269Z","end":"2026-03-26T14:51:33.832027Z","steps":["trace[797805650] 'read index received' (duration: 1.592743246s)","trace[797805650] 'applied index is now lower than readState.Index' (duration: 12.881µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:33.832105Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:31.993308Z","time spent":"1.83879003s","remote":"199.204.45.69:50892","response type":"/etcdserverpb.Cluster/MemberList","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:33.832350Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.852345746s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:33.832397Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.849198647s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-26T14:51:33.832448Z","caller":"traceutil/trace.go:172","msg":"trace[1871176605] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:631; }","duration":"1.852456958s","start":"2026-03-26T14:51:31.979975Z","end":"2026-03-26T14:51:33.832432Z","steps":["trace[1871176605] 'agreement among raft nodes before linearized reading' (duration: 1.852279175s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:33.832473Z","caller":"traceutil/trace.go:172","msg":"trace[1312968436] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649; range_end:; response_count:1; response_revision:631; }","duration":"1.849279318s","start":"2026-03-26T14:51:31.983166Z","end":"2026-03-26T14:51:33.832445Z","steps":["trace[1312968436] 'agreement among raft nodes before linearized reading' (duration: 1.849043044s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:33.832529Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:31.983151Z","time spent":"1.849361419s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":816,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:33.946846Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486956,"received-request-id":10938290401367486953} {"level":"info","ts":"2026-03-26T14:51:34.016637Z","caller":"traceutil/trace.go:172","msg":"trace[659897510] linearizableReadLoop","detail":"{readStateIndex:688; appliedIndex:688; }","duration":"184.451892ms","start":"2026-03-26T14:51:33.832159Z","end":"2026-03-26T14:51:34.016611Z","steps":["trace[659897510] 'read index received' (duration: 184.443722ms)","trace[659897510] 'applied index is now lower than readState.Index' (duration: 6.89µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:34.510144Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"289.205783ms"} {"level":"warn","ts":"2026-03-26T14:51:34.510246Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.263539446s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:34.510378Z","caller":"traceutil/trace.go:172","msg":"trace[2081916309] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:634; }","duration":"1.263657498s","start":"2026-03-26T14:51:33.246691Z","end":"2026-03-26T14:51:34.510348Z","steps":["trace[2081916309] 'agreement among raft nodes before linearized reading' (duration: 770.187075ms)","trace[2081916309] 'range keys from in-memory index tree' (duration: 493.28305ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:34.510458Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:33.246672Z","time spent":"1.26377672s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:34.510476Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"750.891333ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:34.510539Z","caller":"traceutil/trace.go:172","msg":"trace[1013995501] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:634; }","duration":"750.948584ms","start":"2026-03-26T14:51:33.759567Z","end":"2026-03-26T14:51:34.510516Z","steps":["trace[1013995501] 'agreement among raft nodes before linearized reading' (duration: 257.293628ms)","trace[1013995501] 'range keys from in-memory index tree' (duration: 493.574525ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:34.510558Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:33.851218Z","time spent":"659.335849ms","remote":"199.204.45.69:50920","response type":"/etcdserverpb.Maintenance/Status","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:34.510578Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:33.759553Z","time spent":"751.018066ms","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:34.510826Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"563.802121ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:51:34.511139Z","caller":"traceutil/trace.go:172","msg":"trace[46822110] transaction","detail":"{read_only:false; response_revision:636; number_of_response:1; }","duration":"747.867477ms","start":"2026-03-26T14:51:33.763256Z","end":"2026-03-26T14:51:34.511123Z","steps":["trace[46822110] 'process raft request' (duration: 747.666313ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:34.511216Z","caller":"traceutil/trace.go:172","msg":"trace[453508203] linearizableReadLoop","detail":"{readStateIndex:691; appliedIndex:688; }","duration":"494.482562ms","start":"2026-03-26T14:51:34.016716Z","end":"2026-03-26T14:51:34.511198Z","steps":["trace[453508203] 'read index received' (duration: 1.829374ms)","trace[453508203] 'applied index is now lower than readState.Index' (duration: 492.652208ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:34.511174Z","caller":"traceutil/trace.go:172","msg":"trace[557699833] transaction","detail":"{read_only:false; response_revision:635; number_of_response:1; }","duration":"759.121686ms","start":"2026-03-26T14:51:33.751904Z","end":"2026-03-26T14:51:34.511026Z","steps":["trace[557699833] 'process raft request' (duration: 195.074201ms)","trace[557699833] 'compare' (duration: 563.470694ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:34.511247Z","caller":"traceutil/trace.go:172","msg":"trace[972189676] transaction","detail":"{read_only:false; response_revision:637; number_of_response:1; }","duration":"675.98006ms","start":"2026-03-26T14:51:33.835241Z","end":"2026-03-26T14:51:34.511222Z","steps":["trace[972189676] 'process raft request' (duration: 675.778216ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:34.511307Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:33.763244Z","time spent":"747.965569ms","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5124,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:34.511335Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:33.751837Z","time spent":"759.454612ms","remote":"127.0.0.1:36606","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:34.511349Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:33.835222Z","time spent":"676.077021ms","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:34.511337Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"675.790256ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:34.510246Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"460.409392ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:34.511607Z","caller":"traceutil/trace.go:172","msg":"trace[470482852] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:634; }","duration":"461.771407ms","start":"2026-03-26T14:51:34.049808Z","end":"2026-03-26T14:51:34.511579Z","steps":["trace[470482852] 'range keys from in-memory index tree' (duration: 460.35884ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:34.511613Z","caller":"traceutil/trace.go:172","msg":"trace[1627692214] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:638; }","duration":"675.910588ms","start":"2026-03-26T14:51:33.835535Z","end":"2026-03-26T14:51:34.511445Z","steps":["trace[1627692214] 'agreement among raft nodes before linearized reading' (duration: 675.766585ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.016145Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486960,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:35.118841Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"403.29786ms"} {"level":"info","ts":"2026-03-26T14:51:35.118884Z","caller":"traceutil/trace.go:172","msg":"trace[253283530] linearizableReadLoop","detail":"{readStateIndex:692; appliedIndex:692; }","duration":"603.357201ms","start":"2026-03-26T14:51:34.515496Z","end":"2026-03-26T14:51:35.118853Z","steps":["trace[253283530] 'read index received' (duration: 603.344351ms)","trace[253283530] 'applied index is now lower than readState.Index' (duration: 11.04µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:35.457665Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486962,"received-request-id":10938290401367486960} {"level":"warn","ts":"2026-03-26T14:51:35.457652Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"138.724519ms"} {"level":"warn","ts":"2026-03-26T14:51:35.619393Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486962,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:35.794290Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.279502397s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-26T14:51:35.794448Z","caller":"traceutil/trace.go:172","msg":"trace[888227582] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:638; }","duration":"1.27965978s","start":"2026-03-26T14:51:34.514768Z","end":"2026-03-26T14:51:35.794428Z","steps":["trace[888227582] 'agreement among raft nodes before linearized reading' (duration: 604.167576ms)","trace[888227582] 'range keys from in-memory index tree' (duration: 675.219419ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:35.794493Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.514750Z","time spent":"1.279730781s","remote":"127.0.0.1:36858","response type":"/etcdserverpb.KV/Range","request count":0,"request size":51,"response count":1,"response size":464,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:35.794503Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.279080329s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-scheduler-controller-1\" limit:1 ","response":"range_response_count:1 size:4764"} {"level":"info","ts":"2026-03-26T14:51:35.794560Z","caller":"traceutil/trace.go:172","msg":"trace[388953073] range","detail":"{range_begin:/registry/pods/kube-system/kube-scheduler-controller-1; range_end:; response_count:1; response_revision:638; }","duration":"1.27913829s","start":"2026-03-26T14:51:34.515403Z","end":"2026-03-26T14:51:35.794541Z","steps":["trace[388953073] 'agreement among raft nodes before linearized reading' (duration: 603.466442ms)","trace[388953073] 'range keys from in-memory index tree' (duration: 675.530274ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:35.794605Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.515395Z","time spent":"1.279197162s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":1,"response size":4786,"request content":"key:\"/registry/pods/kube-system/kube-scheduler-controller-1\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:35.794969Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"675.907772ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-26T14:51:35.795401Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"696.257845ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:35.795448Z","caller":"traceutil/trace.go:172","msg":"trace[1758183167] transaction","detail":"{read_only:false; response_revision:639; number_of_response:1; }","duration":"1.279497417s","start":"2026-03-26T14:51:34.515871Z","end":"2026-03-26T14:51:35.795369Z","steps":["trace[1758183167] 'process raft request' (duration: 603.113207ms)","trace[1758183167] 'compare' (duration: 675.787129ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:35.795521Z","caller":"traceutil/trace.go:172","msg":"trace[1601844414] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:638; }","duration":"698.036078ms","start":"2026-03-26T14:51:35.097472Z","end":"2026-03-26T14:51:35.795508Z","steps":["trace[1601844414] 'range keys from in-memory index tree' (duration: 696.204654ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.795642Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.515847Z","time spent":"1.27971276s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":879,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-26T14:51:35.823472Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"165.735755ms"} {"level":"info","ts":"2026-03-26T14:51:35.823529Z","caller":"traceutil/trace.go:172","msg":"trace[1426711508] linearizableReadLoop","detail":"{readStateIndex:693; appliedIndex:695; }","duration":"704.564707ms","start":"2026-03-26T14:51:35.118947Z","end":"2026-03-26T14:51:35.823512Z","steps":["trace[1426711508] 'read index received' (duration: 704.557867ms)","trace[1426711508] 'applied index is now lower than readState.Index' (duration: 6.11µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:35.823641Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.01701918s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/prioritylevelconfigurations\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:35.823693Z","caller":"traceutil/trace.go:172","msg":"trace[327067207] range","detail":"{range_begin:/registry/prioritylevelconfigurations; range_end:; response_count:0; response_revision:641; }","duration":"1.017076681s","start":"2026-03-26T14:51:34.806604Z","end":"2026-03-26T14:51:35.823681Z","steps":["trace[327067207] 'agreement among raft nodes before linearized reading' (duration: 1.0169937s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.823735Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.806580Z","time spent":"1.017140722s","remote":"127.0.0.1:37454","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":0,"response size":27,"request content":"key:\"/registry/prioritylevelconfigurations\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:35.823848Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.285471433s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions/controller-3\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:35.823922Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.30370632s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:35.823986Z","caller":"traceutil/trace.go:172","msg":"trace[1697702073] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:641; }","duration":"1.303779951s","start":"2026-03-26T14:51:34.520191Z","end":"2026-03-26T14:51:35.823971Z","steps":["trace[1697702073] 'agreement among raft nodes before linearized reading' (duration: 1.30367749s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.824036Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.520181Z","time spent":"1.303847992s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:51:35.823900Z","caller":"traceutil/trace.go:172","msg":"trace[524751398] range","detail":"{range_begin:/registry/minions/controller-3; range_end:; response_count:0; response_revision:641; }","duration":"1.285524484s","start":"2026-03-26T14:51:34.538363Z","end":"2026-03-26T14:51:35.823887Z","steps":["trace[524751398] 'agreement among raft nodes before linearized reading' (duration: 1.285443092s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:35.824402Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:34.538349Z","time spent":"1.286031663s","remote":"127.0.0.1:36888","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":27,"request content":"key:\"/registry/minions/controller-3\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:36.254456Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486964,"received-request-id":10938290401367486962} {"level":"warn","ts":"2026-03-26T14:51:36.254451Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"230.894541ms"} {"level":"warn","ts":"2026-03-26T14:51:36.324526Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486964,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:36.551255Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"96.697692ms"} {"level":"info","ts":"2026-03-26T14:51:36.551397Z","caller":"traceutil/trace.go:172","msg":"trace[1320586914] linearizableReadLoop","detail":"{readStateIndex:696; appliedIndex:696; }","duration":"727.807457ms","start":"2026-03-26T14:51:35.823567Z","end":"2026-03-26T14:51:36.551375Z","steps":["trace[1320586914] 'read index received' (duration: 727.799357ms)","trace[1320586914] 'applied index is now lower than readState.Index' (duration: 6.66µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:36.551727Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"297.086368ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"warn","ts":"2026-03-26T14:51:36.551791Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"755.255952ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:36.551838Z","caller":"traceutil/trace.go:172","msg":"trace[371926262] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:642; }","duration":"755.358744ms","start":"2026-03-26T14:51:35.796468Z","end":"2026-03-26T14:51:36.551826Z","steps":["trace[371926262] 'agreement among raft nodes before linearized reading' (duration: 755.208711ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.551872Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"751.755116ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-26T14:51:36.551922Z","caller":"traceutil/trace.go:172","msg":"trace[145968947] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649; range_end:; response_count:1; response_revision:642; }","duration":"751.796277ms","start":"2026-03-26T14:51:35.800109Z","end":"2026-03-26T14:51:36.551905Z","steps":["trace[145968947] 'agreement among raft nodes before linearized reading' (duration: 751.711075ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.551957Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.800097Z","time spent":"751.851748ms","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":816,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:36.551969Z","caller":"traceutil/trace.go:172","msg":"trace[1502370930] transaction","detail":"{read_only:false; response_revision:642; number_of_response:1; }","duration":"748.258763ms","start":"2026-03-26T14:51:35.803692Z","end":"2026-03-26T14:51:36.551951Z","steps":["trace[1502370930] 'process raft request' (duration: 450.872549ms)","trace[1502370930] 'compare' (duration: 296.782113ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:36.552035Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.803666Z","time spent":"748.339454ms","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5043,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:36.551778Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"753.367376ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:2 size:263"} {"level":"info","ts":"2026-03-26T14:51:36.552158Z","caller":"traceutil/trace.go:172","msg":"trace[1720659573] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:2; response_revision:642; }","duration":"753.757343ms","start":"2026-03-26T14:51:35.798389Z","end":"2026-03-26T14:51:36.552147Z","steps":["trace[1720659573] 'agreement among raft nodes before linearized reading' (duration: 753.243444ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.552360Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.798340Z","time spent":"753.848955ms","remote":"127.0.0.1:36606","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":2,"response size":285,"request content":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" "} {"level":"warn","ts":"2026-03-26T14:51:36.553449Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486965,"received-request-id":10938290401367486964} {"level":"warn","ts":"2026-03-26T14:51:36.553576Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"355.064677ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/certificatesigningrequests\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:36.553598Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"728.324716ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:36.553630Z","caller":"traceutil/trace.go:172","msg":"trace[1264799496] range","detail":"{range_begin:/registry/certificatesigningrequests; range_end:; response_count:0; response_revision:644; }","duration":"355.098367ms","start":"2026-03-26T14:51:36.198501Z","end":"2026-03-26T14:51:36.553599Z","steps":["trace[1264799496] 'agreement among raft nodes before linearized reading' (duration: 355.039896ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:36.553644Z","caller":"traceutil/trace.go:172","msg":"trace[1128448317] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:644; }","duration":"728.369467ms","start":"2026-03-26T14:51:35.825260Z","end":"2026-03-26T14:51:36.553630Z","steps":["trace[1128448317] 'agreement among raft nodes before linearized reading' (duration: 728.238334ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.553652Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.198479Z","time spent":"355.167078ms","remote":"127.0.0.1:37048","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":0,"response size":27,"request content":"key:\"/registry/certificatesigningrequests\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:36.553672Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:35.825232Z","time spent":"728.433118ms","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:51:36.553706Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"409.782315ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:36.553881Z","caller":"traceutil/trace.go:172","msg":"trace[1446147624] range","detail":"{range_begin:/registry/pods; range_end:; response_count:0; response_revision:644; }","duration":"409.868796ms","start":"2026-03-26T14:51:36.143892Z","end":"2026-03-26T14:51:36.553761Z","steps":["trace[1446147624] 'agreement among raft nodes before linearized reading' (duration: 409.662352ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:36.553964Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.143869Z","time spent":"410.082861ms","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/pods\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:37.096750Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486969,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:37.597117Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486969,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:37.726814Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"628.800463ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:37.726904Z","caller":"traceutil/trace.go:172","msg":"trace[1801628651] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:645; }","duration":"628.911585ms","start":"2026-03-26T14:51:37.097974Z","end":"2026-03-26T14:51:37.726886Z","steps":["trace[1801628651] 'range keys from in-memory index tree' (duration: 628.730861ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.727352Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.019790783s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:51:37.727409Z","caller":"traceutil/trace.go:172","msg":"trace[551675776] transaction","detail":"{read_only:false; response_revision:646; number_of_response:1; }","duration":"1.132636441s","start":"2026-03-26T14:51:36.594761Z","end":"2026-03-26T14:51:37.727397Z","steps":["trace[551675776] 'process raft request' (duration: 112.755147ms)","trace[551675776] 'compare' (duration: 1.019454627s)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:37.727452Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.594733Z","time spent":"1.132701142s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":812,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-26T14:51:37.777529Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.070056223s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-26T14:51:37.777727Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"870.365338ms"} {"level":"info","ts":"2026-03-26T14:51:37.777831Z","caller":"traceutil/trace.go:172","msg":"trace[185876883] linearizableReadLoop","detail":"{readStateIndex:700; appliedIndex:701; }","duration":"1.181986165s","start":"2026-03-26T14:51:36.595825Z","end":"2026-03-26T14:51:37.777811Z","steps":["trace[185876883] 'read index received' (duration: 1.181979495s)","trace[185876883] 'applied index is now lower than readState.Index' (duration: 5.29µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:37.778072Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.182230859s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.157\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-26T14:51:37.778104Z","caller":"traceutil/trace.go:172","msg":"trace[727914223] range","detail":"{range_begin:/registry/masterleases/199.204.45.157; range_end:; response_count:1; response_revision:646; }","duration":"1.18228722s","start":"2026-03-26T14:51:36.595809Z","end":"2026-03-26T14:51:37.778097Z","steps":["trace[727914223] 'agreement among raft nodes before linearized reading' (duration: 1.182093247s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.778133Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.595783Z","time spent":"1.18233817s","remote":"127.0.0.1:36606","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":157,"request content":"key:\"/registry/masterleases/199.204.45.157\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:37.778764Z","caller":"traceutil/trace.go:172","msg":"trace[1435691286] transaction","detail":"{read_only:false; response_revision:649; number_of_response:1; }","duration":"1.176129798s","start":"2026-03-26T14:51:36.602602Z","end":"2026-03-26T14:51:37.778731Z","steps":["trace[1435691286] 'process raft request' (duration: 1.176042027s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.778973Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.602580Z","time spent":"1.176283981s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7455,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:51:37.779525Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367486972,"received-request-id":10938290401367486969} {"level":"warn","ts":"2026-03-26T14:51:37.780781Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"851.941794ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumes\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:37.780890Z","caller":"traceutil/trace.go:172","msg":"trace[1319719244] range","detail":"{range_begin:/registry/persistentvolumes; range_end:; response_count:0; response_revision:649; }","duration":"852.061556ms","start":"2026-03-26T14:51:36.928809Z","end":"2026-03-26T14:51:37.780871Z","steps":["trace[1319719244] 'agreement among raft nodes before linearized reading' (duration: 851.899323ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.780945Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"259.113245ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:37.781004Z","caller":"traceutil/trace.go:172","msg":"trace[1886701084] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:649; }","duration":"259.171466ms","start":"2026-03-26T14:51:37.521818Z","end":"2026-03-26T14:51:37.780989Z","steps":["trace[1886701084] 'agreement among raft nodes before linearized reading' (duration: 259.076575ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:37.781082Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:36.928781Z","time spent":"852.154728ms","remote":"127.0.0.1:36830","response type":"/etcdserverpb.KV/Range","request count":0,"request size":31,"response count":0,"response size":27,"request content":"key:\"/registry/persistentvolumes\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:38.287511Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367486975,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:51:38.529143Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"551.162547ms"} {"level":"warn","ts":"2026-03-26T14:51:38.571300Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"473.096925ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:38.571404Z","caller":"traceutil/trace.go:172","msg":"trace[1960071994] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:649; }","duration":"473.210567ms","start":"2026-03-26T14:51:38.098177Z","end":"2026-03-26T14:51:38.571388Z","steps":["trace[1960071994] 'range keys from in-memory index tree' (duration: 473.038704ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:38.571643Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.779990Z","time spent":"791.645505ms","remote":"127.0.0.1:36606","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-26T14:51:38.583828Z","caller":"traceutil/trace.go:172","msg":"trace[173018597] linearizableReadLoop","detail":"{readStateIndex:704; appliedIndex:705; }","duration":"796.42092ms","start":"2026-03-26T14:51:37.787381Z","end":"2026-03-26T14:51:38.583802Z","steps":["trace[173018597] 'read index received' (duration: 796.40533ms)","trace[173018597] 'applied index is now lower than readState.Index' (duration: 13.95µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:38.584799Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"797.401778ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-apiserver-controller-1\" limit:1 ","response":"range_response_count:1 size:7715"} {"level":"info","ts":"2026-03-26T14:51:38.584908Z","caller":"traceutil/trace.go:172","msg":"trace[1267020066] range","detail":"{range_begin:/registry/pods/kube-system/kube-apiserver-controller-1; range_end:; response_count:1; response_revision:649; }","duration":"797.521ms","start":"2026-03-26T14:51:37.787371Z","end":"2026-03-26T14:51:38.584892Z","steps":["trace[1267020066] 'agreement among raft nodes before linearized reading' (duration: 796.574243ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:38.584949Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.787359Z","time spent":"797.580242ms","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":1,"response size":7737,"request content":"key:\"/registry/pods/kube-system/kube-apiserver-controller-1\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:38.585207Z","caller":"traceutil/trace.go:172","msg":"trace[1802038145] transaction","detail":"{read_only:false; response_revision:651; number_of_response:1; }","duration":"800.199169ms","start":"2026-03-26T14:51:37.784997Z","end":"2026-03-26T14:51:38.585196Z","steps":["trace[1802038145] 'process raft request' (duration: 800.013936ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:38.585209Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"5f170c75a1d97cc switched to configuration voters=(428247441089599436 13087414626088587099) learners=(6735209715263037861)"} {"level":"warn","ts":"2026-03-26T14:51:38.585286Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.784979Z","time spent":"800.261371ms","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":778,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:51:38.585329Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"44a3176ec85feb9","local-member-id":"5f170c75a1d97cc","added-peer-id":"5d78458d260361a5","added-peer-peer-urls":["https://199.204.45.69:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-26T14:51:38.585373Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.585398Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.585544Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.585585Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5","remote-peer-urls":["https://199.204.45.69:2380"]} {"level":"info","ts":"2026-03-26T14:51:38.585613Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.585627Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.585642Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.585916Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:38.588339Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"5f170c75a1d97cc","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"5d78458d260361a5"} {"level":"warn","ts":"2026-03-26T14:51:38.588589Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:37.797382Z","time spent":"791.200575ms","remote":"199.204.45.69:50976","response type":"/etcdserverpb.Cluster/MemberAdd","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-26T14:51:38.648121Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"126.555478ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:38.648200Z","caller":"traceutil/trace.go:172","msg":"trace[705038634] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:652; }","duration":"126.644499ms","start":"2026-03-26T14:51:38.521540Z","end":"2026-03-26T14:51:38.648185Z","steps":["trace[705038634] 'agreement among raft nodes before linearized reading' (duration: 126.510367ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:38.771798Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"180.815085ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-26T14:51:38.771854Z","caller":"traceutil/trace.go:172","msg":"trace[1918538613] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649; range_end:; response_count:1; response_revision:653; }","duration":"180.881557ms","start":"2026-03-26T14:51:38.590958Z","end":"2026-03-26T14:51:38.771839Z","steps":["trace[1918538613] 'agreement among raft nodes before linearized reading' (duration: 124.026613ms)","trace[1918538613] 'range keys from in-memory index tree' (duration: 56.566058ms)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:38.772041Z","caller":"traceutil/trace.go:172","msg":"trace[900304030] transaction","detail":"{read_only:false; response_revision:654; number_of_response:1; }","duration":"174.245107ms","start":"2026-03-26T14:51:38.597789Z","end":"2026-03-26T14:51:38.772034Z","steps":["trace[900304030] 'process raft request' (duration: 116.197332ms)","trace[900304030] 'compare' (duration: 57.656257ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:38.772033Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"122.277171ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:38.772079Z","caller":"traceutil/trace.go:172","msg":"trace[491981135] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:655; }","duration":"122.330782ms","start":"2026-03-26T14:51:38.649738Z","end":"2026-03-26T14:51:38.772069Z","steps":["trace[491981135] 'agreement among raft nodes before linearized reading' (duration: 122.259511ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:38.772081Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"181.397346ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:442"} {"level":"info","ts":"2026-03-26T14:51:38.772493Z","caller":"traceutil/trace.go:172","msg":"trace[1959079040] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:653; }","duration":"181.808533ms","start":"2026-03-26T14:51:38.590671Z","end":"2026-03-26T14:51:38.772480Z","steps":["trace[1959079040] 'agreement among raft nodes before linearized reading' (duration: 124.330968ms)","trace[1959079040] 'range keys from in-memory index tree' (duration: 56.799893ms)"],"step_count":2} {"level":"error","ts":"2026-03-26T14:51:38.850813Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*peerMemberPromoteHandler).ServeHTTP\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/peer.go:140\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-26T14:51:38.850915Z","caller":"etcdhttp/peer.go:152","msg":"failed to promote a member","member-id":"5d78458d260361a5","error":"etcdserver: can only promote a learner member which is in sync with leader"} {"level":"info","ts":"2026-03-26T14:51:39.104160Z","caller":"traceutil/trace.go:172","msg":"trace[1538957536] linearizableReadLoop","detail":"{readStateIndex:719; appliedIndex:719; }","duration":"116.609747ms","start":"2026-03-26T14:51:38.987534Z","end":"2026-03-26T14:51:39.104144Z","steps":["trace[1538957536] 'read index received' (duration: 116.605987ms)","trace[1538957536] 'applied index is now lower than readState.Index' (duration: 2.88µs)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:51:39.160493Z","caller":"traceutil/trace.go:172","msg":"trace[1491846420] transaction","detail":"{read_only:false; response_revision:663; number_of_response:1; }","duration":"173.492055ms","start":"2026-03-26T14:51:38.986985Z","end":"2026-03-26T14:51:39.160477Z","steps":["trace[1491846420] 'process raft request' (duration: 116.047999ms)","trace[1491846420] 'compare' (duration: 57.342834ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:39.160564Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"173.005465ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-controller-manager-controller-1\" limit:1 ","response":"range_response_count:1 size:7470"} {"level":"info","ts":"2026-03-26T14:51:39.160647Z","caller":"traceutil/trace.go:172","msg":"trace[1133537542] range","detail":"{range_begin:/registry/pods/kube-system/kube-controller-manager-controller-1; range_end:; response_count:1; response_revision:662; }","duration":"173.094977ms","start":"2026-03-26T14:51:38.987531Z","end":"2026-03-26T14:51:39.160626Z","steps":["trace[1133537542] 'agreement among raft nodes before linearized reading' (duration: 116.675278ms)","trace[1133537542] 'range keys from in-memory index tree' (duration: 56.164315ms)"],"step_count":2} {"level":"error","ts":"2026-03-26T14:51:39.289676Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"info","ts":"2026-03-26T14:51:39.442282Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.442340Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.442371Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.454742Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"5f170c75a1d97cc","to":"5d78458d260361a5","stream-type":"stream Message"} {"level":"info","ts":"2026-03-26T14:51:39.454781Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.459658Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"5f170c75a1d97cc","to":"5d78458d260361a5","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-26T14:51:39.459680Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.486374Z","caller":"etcdserver/server.go:1838","msg":"sending merged snapshot","from":"5f170c75a1d97cc","to":"5d78458d260361a5","bytes":1172430,"size":"1.2 MB"} {"level":"info","ts":"2026-03-26T14:51:39.486881Z","caller":"rafthttp/snapshot_sender.go:82","msg":"sending database snapshot","snapshot-index":729,"remote-peer-id":"5d78458d260361a5","bytes":1172430,"size":"1.2 MB"} {"level":"info","ts":"2026-03-26T14:51:39.493445Z","caller":"etcdserver/snapshot_merge.go:64","msg":"sent database snapshot to writer","bytes":1163264,"size":"1.2 MB"} {"level":"info","ts":"2026-03-26T14:51:39.506902Z","caller":"rafthttp/snapshot_sender.go:131","msg":"sent database snapshot","snapshot-index":729,"remote-peer-id":"5d78458d260361a5","bytes":1172430,"size":"1.2 MB"} {"level":"warn","ts":"2026-03-26T14:51:39.521271Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:51:39.521291Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5","error":"EOF"} {"level":"info","ts":"2026-03-26T14:51:39.536406Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.538250Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.540457Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"5f170c75a1d97cc","to":"5d78458d260361a5","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-26T14:51:39.540477Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.540517Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.542354Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"5f170c75a1d97cc","to":"5d78458d260361a5","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-26T14:51:39.542394Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.542406Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"5f170c75a1d97cc","remote-peer-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.771125Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"5f170c75a1d97cc switched to configuration voters=(428247441089599436 6735209715263037861 13087414626088587099)"} {"level":"info","ts":"2026-03-26T14:51:39.771470Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"44a3176ec85feb9","local-member-id":"5f170c75a1d97cc","promoted-member-id":"5d78458d260361a5"} {"level":"info","ts":"2026-03-26T14:51:39.771553Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"5f170c75a1d97cc","raft-conf-change":"ConfChangeAddNode","raft-conf-change-node-id":"5d78458d260361a5"} {"level":"warn","ts":"2026-03-26T14:51:41.835357Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.69:52032","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:51:42.511538Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"269.639049ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:51:42.511740Z","caller":"traceutil/trace.go:172","msg":"trace[2038146182] linearizableReadLoop","detail":"{readStateIndex:742; appliedIndex:741; }","duration":"270.494345ms","start":"2026-03-26T14:51:42.241219Z","end":"2026-03-26T14:51:42.511714Z","steps":["trace[2038146182] 'read index received' (duration: 1.567108ms)","trace[2038146182] 'applied index is now lower than readState.Index' (duration: 268.926017ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:42.511912Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"270.684278ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattributesclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:42.511945Z","caller":"traceutil/trace.go:172","msg":"trace[18545785] range","detail":"{range_begin:/registry/volumeattributesclasses; range_end:; response_count:0; response_revision:684; }","duration":"270.730989ms","start":"2026-03-26T14:51:42.241204Z","end":"2026-03-26T14:51:42.511935Z","steps":["trace[18545785] 'agreement among raft nodes before linearized reading' (duration: 270.656288ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:45.864987Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.69:52086","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:51:46.020023Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.666139ms"} {"level":"warn","ts":"2026-03-26T14:51:46.020136Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"5d78458d260361a5","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.793821ms"} {"level":"warn","ts":"2026-03-26T14:51:46.193859Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"173.639572ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:51:46.194134Z","caller":"traceutil/trace.go:172","msg":"trace[738368175] linearizableReadLoop","detail":"{readStateIndex:754; appliedIndex:753; }","duration":"367.268202ms","start":"2026-03-26T14:51:45.826832Z","end":"2026-03-26T14:51:46.194100Z","steps":["trace[738368175] 'read index received' (duration: 194.756841ms)","trace[738368175] 'applied index is now lower than readState.Index' (duration: 172.508521ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:46.194900Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"368.051496ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-26T14:51:46.194962Z","caller":"traceutil/trace.go:172","msg":"trace[1405242099] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:696; }","duration":"368.122317ms","start":"2026-03-26T14:51:45.826825Z","end":"2026-03-26T14:51:46.194947Z","steps":["trace[1405242099] 'agreement among raft nodes before linearized reading' (duration: 367.405225ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:46.194998Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:45.826800Z","time spent":"368.189798ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:46.362022Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"162.491531ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"info","ts":"2026-03-26T14:51:46.362446Z","caller":"traceutil/trace.go:172","msg":"trace[1264542033] linearizableReadLoop","detail":"{readStateIndex:757; appliedIndex:756; }","duration":"168.210508ms","start":"2026-03-26T14:51:46.194200Z","end":"2026-03-26T14:51:46.362410Z","steps":["trace[1264542033] 'read index received' (duration: 17.908577ms)","trace[1264542033] 'applied index is now lower than readState.Index' (duration: 150.298791ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:46.498086Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"400.824798ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:46.498194Z","caller":"traceutil/trace.go:172","msg":"trace[69003810] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:696; }","duration":"400.94446ms","start":"2026-03-26T14:51:46.097226Z","end":"2026-03-26T14:51:46.498170Z","steps":["trace[69003810] 'agreement among raft nodes before linearized reading' (duration: 265.290968ms)","trace[69003810] 'range keys from in-memory index tree' (duration: 135.481249ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:46.498176Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"135.922127ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"warn","ts":"2026-03-26T14:51:46.498282Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"416.957685ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllerrevisions\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:51:46.498210Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"540.051245ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:46.498349Z","caller":"traceutil/trace.go:172","msg":"trace[1266372681] range","detail":"{range_begin:/registry/controllerrevisions; range_end:; response_count:0; response_revision:696; }","duration":"417.012105ms","start":"2026-03-26T14:51:46.081312Z","end":"2026-03-26T14:51:46.498324Z","steps":["trace[1266372681] 'agreement among raft nodes before linearized reading' (duration: 281.218812ms)","trace[1266372681] 'range keys from in-memory index tree' (duration: 135.718072ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:46.498403Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:46.081282Z","time spent":"417.110657ms","remote":"127.0.0.1:37544","response type":"/etcdserverpb.KV/Range","request count":0,"request size":33,"response count":0,"response size":27,"request content":"key:\"/registry/controllerrevisions\" limit:1 "} {"level":"info","ts":"2026-03-26T14:51:46.498401Z","caller":"traceutil/trace.go:172","msg":"trace[370440078] range","detail":"{range_begin:/registry/horizontalpodautoscalers; range_end:; response_count:0; response_revision:696; }","duration":"540.230788ms","start":"2026-03-26T14:51:45.958120Z","end":"2026-03-26T14:51:46.498350Z","steps":["trace[370440078] 'agreement among raft nodes before linearized reading' (duration: 404.421964ms)","trace[370440078] 'range keys from in-memory index tree' (duration: 135.58745ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:46.498751Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:51:45.958090Z","time spent":"540.639335ms","remote":"127.0.0.1:36984","response type":"/etcdserverpb.KV/Range","request count":0,"request size":38,"response count":0,"response size":27,"request content":"key:\"/registry/horizontalpodautoscalers\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:51:46.785165Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"123.396441ms","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:39"} {"level":"info","ts":"2026-03-26T14:51:46.785353Z","caller":"traceutil/trace.go:172","msg":"trace[464817221] linearizableReadLoop","detail":"{readStateIndex:760; appliedIndex:759; }","duration":"123.450032ms","start":"2026-03-26T14:51:46.661886Z","end":"2026-03-26T14:51:46.785336Z","steps":["trace[464817221] 'read index received' (duration: 1.275873ms)","trace[464817221] 'applied index is now lower than readState.Index' (duration: 122.172609ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:46.785542Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"123.644645ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-26T14:51:46.785589Z","caller":"traceutil/trace.go:172","msg":"trace[1333637702] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:696; }","duration":"123.699176ms","start":"2026-03-26T14:51:46.661880Z","end":"2026-03-26T14:51:46.785579Z","steps":["trace[1333637702] 'agreement among raft nodes before linearized reading' (duration: 123.534113ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:51.276422Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"102.638894ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"warn","ts":"2026-03-26T14:51:52.177858Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"170.281691ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:51:52.178023Z","caller":"traceutil/trace.go:172","msg":"trace[942353222] linearizableReadLoop","detail":"{readStateIndex:788; appliedIndex:787; }","duration":"168.499502ms","start":"2026-03-26T14:51:52.009509Z","end":"2026-03-26T14:51:52.178008Z","steps":["trace[942353222] 'read index received' (duration: 699.271µs)","trace[942353222] 'applied index is now lower than readState.Index' (duration: 167.799221ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:52.178341Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"168.825087ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:483"} {"level":"info","ts":"2026-03-26T14:51:52.178395Z","caller":"traceutil/trace.go:172","msg":"trace[212929125] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:720; }","duration":"168.884968ms","start":"2026-03-26T14:51:52.009498Z","end":"2026-03-26T14:51:52.178383Z","steps":["trace[212929125] 'agreement among raft nodes before linearized reading' (duration: 168.601353ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:52.434271Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"107.80517ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:51:52.434443Z","caller":"traceutil/trace.go:172","msg":"trace[1098290178] transaction","detail":"{read_only:false; response_revision:725; number_of_response:1; }","duration":"115.386925ms","start":"2026-03-26T14:51:52.319045Z","end":"2026-03-26T14:51:52.434432Z","steps":["trace[1098290178] 'process raft request' (duration: 115.361284ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:52.434534Z","caller":"traceutil/trace.go:172","msg":"trace[2043120080] transaction","detail":"{read_only:false; response_revision:724; number_of_response:1; }","duration":"116.695537ms","start":"2026-03-26T14:51:52.317821Z","end":"2026-03-26T14:51:52.434517Z","steps":["trace[2043120080] 'process raft request' (duration: 116.539655ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:51:52.653674Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"132.458787ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:51:52.653735Z","caller":"traceutil/trace.go:172","msg":"trace[651295450] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:727; }","duration":"132.527958ms","start":"2026-03-26T14:51:52.521197Z","end":"2026-03-26T14:51:52.653725Z","steps":["trace[651295450] 'agreement among raft nodes before linearized reading' (duration: 132.399236ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:52.653526Z","caller":"traceutil/trace.go:172","msg":"trace[289994785] linearizableReadLoop","detail":"{readStateIndex:794; appliedIndex:793; }","duration":"132.02534ms","start":"2026-03-26T14:51:52.521211Z","end":"2026-03-26T14:51:52.653236Z","steps":["trace[289994785] 'read index received' (duration: 115.592198ms)","trace[289994785] 'applied index is now lower than readState.Index' (duration: 16.432132ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:51:52.655130Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"109.200923ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-26T14:51:52.655205Z","caller":"traceutil/trace.go:172","msg":"trace[561943338] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:727; }","duration":"109.285495ms","start":"2026-03-26T14:51:52.545905Z","end":"2026-03-26T14:51:52.655190Z","steps":["trace[561943338] 'agreement among raft nodes before linearized reading' (duration: 109.099881ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:51:56.623573Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"info","ts":"2026-03-26T14:52:09.507244Z","caller":"etcdserver/server.go:1856","msg":"sent merged snapshot","from":"5f170c75a1d97cc","to":"5d78458d260361a5","bytes":1172430,"size":"1.2 MB","took":"30.02086928s"} {"level":"info","ts":"2026-03-26T14:52:34.570346Z","caller":"traceutil/trace.go:172","msg":"trace[1483697866] transaction","detail":"{read_only:false; response_revision:1004; number_of_response:1; }","duration":"134.549159ms","start":"2026-03-26T14:52:34.435778Z","end":"2026-03-26T14:52:34.570327Z","steps":["trace[1483697866] 'process raft request' (duration: 129.841305ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:34.570951Z","caller":"traceutil/trace.go:172","msg":"trace[287417052] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1005; }","duration":"114.825041ms","start":"2026-03-26T14:52:34.456089Z","end":"2026-03-26T14:52:34.570914Z","steps":["trace[287417052] 'process raft request' (duration: 114.266043ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.074025Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367487446,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:52:35.214577Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"399.272575ms"} {"level":"warn","ts":"2026-03-26T14:52:35.214639Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"5d78458d260361a5","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"399.342156ms"} {"level":"info","ts":"2026-03-26T14:52:35.214684Z","caller":"traceutil/trace.go:172","msg":"trace[930900930] linearizableReadLoop","detail":"{readStateIndex:1163; appliedIndex:1164; }","duration":"641.137395ms","start":"2026-03-26T14:52:34.573523Z","end":"2026-03-26T14:52:35.214660Z","steps":["trace[930900930] 'read index received' (duration: 641.130775ms)","trace[930900930] 'applied index is now lower than readState.Index' (duration: 5.24µs)"],"step_count":2} {"level":"info","ts":"2026-03-26T14:52:35.227590Z","caller":"traceutil/trace.go:172","msg":"trace[362932282] transaction","detail":"{read_only:false; response_revision:1007; number_of_response:1; }","duration":"648.586486ms","start":"2026-03-26T14:52:34.578985Z","end":"2026-03-26T14:52:35.227571Z","steps":["trace[362932282] 'process raft request' (duration: 635.924654ms)","trace[362932282] 'compare' (duration: 12.409148ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:35.227706Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.578967Z","time spent":"648.685528ms","remote":"127.0.0.1:36638","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":7693,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:52:35.227397Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"653.858338ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:35.228107Z","caller":"traceutil/trace.go:172","msg":"trace[1734311250] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1006; }","duration":"654.579107ms","start":"2026-03-26T14:52:34.573511Z","end":"2026-03-26T14:52:35.228090Z","steps":["trace[1734311250] 'agreement among raft nodes before linearized reading' (duration: 641.253487ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.228163Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.573494Z","time spent":"654.658808ms","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:52:35.228238Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367487450,"received-request-id":10938290401367487446} {"level":"info","ts":"2026-03-26T14:52:35.228931Z","caller":"traceutil/trace.go:172","msg":"trace[1338038744] transaction","detail":"{read_only:false; response_revision:1009; number_of_response:1; }","duration":"540.941872ms","start":"2026-03-26T14:52:34.687978Z","end":"2026-03-26T14:52:35.228920Z","steps":["trace[1338038744] 'process raft request' (duration: 540.78581ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.229023Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.687945Z","time spent":"541.036983ms","remote":"127.0.0.1:36638","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":73770,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-26T14:52:35.229133Z","caller":"traceutil/trace.go:172","msg":"trace[1710709177] transaction","detail":"{read_only:false; response_revision:1008; number_of_response:1; }","duration":"574.883103ms","start":"2026-03-26T14:52:34.654237Z","end":"2026-03-26T14:52:35.229120Z","steps":["trace[1710709177] 'process raft request' (duration: 574.254485ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.229197Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.654209Z","time spent":"574.950765ms","remote":"127.0.0.1:36638","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":151212,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:52:35.230365Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"604.295983ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ingressclasses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:35.230460Z","caller":"traceutil/trace.go:172","msg":"trace[1626417356] range","detail":"{range_begin:/registry/ingressclasses; range_end:; response_count:0; response_revision:1009; }","duration":"604.398364ms","start":"2026-03-26T14:52:34.626049Z","end":"2026-03-26T14:52:35.230447Z","steps":["trace[1626417356] 'agreement among raft nodes before linearized reading' (duration: 604.261053ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.230487Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.626030Z","time spent":"604.451545ms","remote":"127.0.0.1:37142","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":27,"request content":"key:\"/registry/ingressclasses\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:35.230689Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"130.134884ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:35.230717Z","caller":"traceutil/trace.go:172","msg":"trace[1132144868] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1009; }","duration":"130.163504ms","start":"2026-03-26T14:52:35.100548Z","end":"2026-03-26T14:52:35.230711Z","steps":["trace[1132144868] 'agreement among raft nodes before linearized reading' (duration: 130.128333ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.230749Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"515.0566ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cronjobs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:35.230836Z","caller":"traceutil/trace.go:172","msg":"trace[1893106668] range","detail":"{range_begin:/registry/cronjobs; range_end:; response_count:0; response_revision:1009; }","duration":"515.142641ms","start":"2026-03-26T14:52:34.715682Z","end":"2026-03-26T14:52:35.230824Z","steps":["trace[1893106668] 'agreement among raft nodes before linearized reading' (duration: 515.04048ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:35.230866Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:34.715658Z","time spent":"515.199652ms","remote":"127.0.0.1:37024","response type":"/etcdserverpb.KV/Range","request count":0,"request size":22,"response count":0,"response size":27,"request content":"key:\"/registry/cronjobs\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:36.058211Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"201.010809ms"} {"level":"warn","ts":"2026-03-26T14:52:36.058305Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"5d78458d260361a5","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"201.114091ms"} {"level":"info","ts":"2026-03-26T14:52:36.076818Z","caller":"traceutil/trace.go:172","msg":"trace[1012317493] transaction","detail":"{read_only:false; response_revision:1027; number_of_response:1; }","duration":"392.658135ms","start":"2026-03-26T14:52:35.684131Z","end":"2026-03-26T14:52:36.076789Z","steps":["trace[1012317493] 'process raft request' (duration: 374.313917ms)","trace[1012317493] 'compare' (duration: 17.519977ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:36.076943Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:35.684108Z","time spent":"392.783047ms","remote":"127.0.0.1:36638","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":14661,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-26T14:52:36.078751Z","caller":"traceutil/trace.go:172","msg":"trace[975225276] linearizableReadLoop","detail":"{readStateIndex:1202; appliedIndex:1202; }","duration":"205.50507ms","start":"2026-03-26T14:52:35.873214Z","end":"2026-03-26T14:52:36.078719Z","steps":["trace[975225276] 'read index received' (duration: 205.49182ms)","trace[975225276] 'applied index is now lower than readState.Index' (duration: 11.12µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:36.079058Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"205.828856ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-26T14:52:36.079236Z","caller":"traceutil/trace.go:172","msg":"trace[1296344102] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1027; }","duration":"205.889227ms","start":"2026-03-26T14:52:35.873197Z","end":"2026-03-26T14:52:36.079086Z","steps":["trace[1296344102] 'agreement among raft nodes before linearized reading' (duration: 205.682093ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:36.081841Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"185.963166ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-26T14:52:36.081902Z","caller":"traceutil/trace.go:172","msg":"trace[1510870530] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1028; }","duration":"186.041107ms","start":"2026-03-26T14:52:35.895848Z","end":"2026-03-26T14:52:36.081889Z","steps":["trace[1510870530] 'agreement among raft nodes before linearized reading' (duration: 185.864994ms)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:36.082601Z","caller":"traceutil/trace.go:172","msg":"trace[1461213828] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1028; }","duration":"286.732731ms","start":"2026-03-26T14:52:35.795846Z","end":"2026-03-26T14:52:36.082578Z","steps":["trace[1461213828] 'process raft request' (duration: 286.310016ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:41.929032Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"268.290005ms"} {"level":"warn","ts":"2026-03-26T14:52:41.929123Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"5d78458d260361a5","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"268.388586ms"} {"level":"info","ts":"2026-03-26T14:52:41.965026Z","caller":"traceutil/trace.go:172","msg":"trace[769005537] transaction","detail":"{read_only:false; response_revision:1071; number_of_response:1; }","duration":"500.541183ms","start":"2026-03-26T14:52:41.464461Z","end":"2026-03-26T14:52:41.965002Z","steps":["trace[769005537] 'process raft request' (duration: 464.914981ms)","trace[769005537] 'compare' (duration: 35.324518ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:41.965173Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:41.464443Z","time spent":"500.661055ms","remote":"127.0.0.1:36718","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":1036285,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:52:41.967591Z","caller":"traceutil/trace.go:172","msg":"trace[1396640423] linearizableReadLoop","detail":"{readStateIndex:1256; appliedIndex:1256; }","duration":"423.402511ms","start":"2026-03-26T14:52:41.544167Z","end":"2026-03-26T14:52:41.967569Z","steps":["trace[1396640423] 'read index received' (duration: 423.396301ms)","trace[1396640423] 'applied index is now lower than readState.Index' (duration: 4.69µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:42.118316Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"150.396809ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:27"} {"level":"info","ts":"2026-03-26T14:52:42.118529Z","caller":"traceutil/trace.go:172","msg":"trace[261642889] linearizableReadLoop","detail":"{readStateIndex:1259; appliedIndex:1258; }","duration":"150.870695ms","start":"2026-03-26T14:52:41.967641Z","end":"2026-03-26T14:52:42.118512Z","steps":["trace[261642889] 'read index received' (duration: 1.028223ms)","trace[261642889] 'applied index is now lower than readState.Index' (duration: 149.841632ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:42.118697Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"325.308074ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-26T14:52:42.118725Z","caller":"traceutil/trace.go:172","msg":"trace[1556206263] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1073; }","duration":"325.342215ms","start":"2026-03-26T14:52:41.793373Z","end":"2026-03-26T14:52:42.118716Z","steps":["trace[1556206263] 'agreement among raft nodes before linearized reading' (duration: 325.226782ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:42.118755Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:41.793348Z","time spent":"325.399695ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:42.120456Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"576.279293ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces/kube-system\" limit:1 ","response":"range_response_count:1 size:351"} {"level":"info","ts":"2026-03-26T14:52:42.120525Z","caller":"traceutil/trace.go:172","msg":"trace[275295752] range","detail":"{range_begin:/registry/namespaces/kube-system; range_end:; response_count:1; response_revision:1073; }","duration":"576.360594ms","start":"2026-03-26T14:52:41.544149Z","end":"2026-03-26T14:52:42.120509Z","steps":["trace[275295752] 'agreement among raft nodes before linearized reading' (duration: 423.789176ms)","trace[275295752] 'range keys from in-memory index tree' (duration: 150.716464ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:42.120556Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:41.544124Z","time spent":"576.425074ms","remote":"127.0.0.1:36736","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":1,"response size":373,"request content":"key:\"/registry/namespaces/kube-system\" limit:1 "} {"level":"info","ts":"2026-03-26T14:52:42.290477Z","caller":"traceutil/trace.go:172","msg":"trace[1063393039] linearizableReadLoop","detail":"{readStateIndex:1260; appliedIndex:1260; }","duration":"171.890584ms","start":"2026-03-26T14:52:42.118560Z","end":"2026-03-26T14:52:42.290451Z","steps":["trace[1063393039] 'read index received' (duration: 171.882714ms)","trace[1063393039] 'applied index is now lower than readState.Index' (duration: 6.5µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:43.269555Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.172075398s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:43.269974Z","caller":"traceutil/trace.go:172","msg":"trace[249097789] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1074; }","duration":"1.172489853s","start":"2026-03-26T14:52:42.097461Z","end":"2026-03-26T14:52:43.269950Z","steps":["trace[249097789] 'agreement among raft nodes before linearized reading' (duration: 193.339166ms)","trace[249097789] 'range keys from in-memory index tree' (duration: 978.713002ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:43.270180Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.185429566s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-8sdn4\" limit:1 ","response":"range_response_count:1 size:18354"} {"level":"info","ts":"2026-03-26T14:52:43.270283Z","caller":"traceutil/trace.go:172","msg":"trace[671126729] range","detail":"{range_begin:/registry/pods/kube-system/cilium-8sdn4; range_end:; response_count:1; response_revision:1074; }","duration":"1.185539987s","start":"2026-03-26T14:52:42.084722Z","end":"2026-03-26T14:52:43.270262Z","steps":["trace[671126729] 'agreement among raft nodes before linearized reading' (duration: 206.095796ms)","trace[671126729] 'range keys from in-memory index tree' (duration: 979.216508ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:43.270325Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:42.084699Z","time spent":"1.185617039s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18376,"request content":"key:\"/registry/pods/kube-system/cilium-8sdn4\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:43.270687Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.011199371s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:52:43.271171Z","caller":"traceutil/trace.go:172","msg":"trace[618542495] linearizableReadLoop","detail":"{readStateIndex:1264; appliedIndex:1260; }","duration":"980.600216ms","start":"2026-03-26T14:52:42.290558Z","end":"2026-03-26T14:52:43.271159Z","steps":["trace[618542495] 'read index received' (duration: 1.167695ms)","trace[618542495] 'applied index is now lower than readState.Index' (duration: 979.431741ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:43.271403Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.143677772s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/kube-system/cilium-config\" limit:1 ","response":"range_response_count:1 size:6534"} {"level":"info","ts":"2026-03-26T14:52:43.271439Z","caller":"traceutil/trace.go:172","msg":"trace[774375077] range","detail":"{range_begin:/registry/configmaps/kube-system/cilium-config; range_end:; response_count:1; response_revision:1079; }","duration":"1.143715083s","start":"2026-03-26T14:52:42.127716Z","end":"2026-03-26T14:52:43.271431Z","steps":["trace[774375077] 'agreement among raft nodes before linearized reading' (duration: 1.143615752s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:43.271427Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.145105562s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:52:43.271463Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:42.127704Z","time spent":"1.143751734s","remote":"127.0.0.1:36722","response type":"/etcdserverpb.KV/Range","request count":0,"request size":50,"response count":1,"response size":6556,"request content":"key:\"/registry/configmaps/kube-system/cilium-config\" limit:1 "} {"level":"info","ts":"2026-03-26T14:52:43.271479Z","caller":"traceutil/trace.go:172","msg":"trace[1137928152] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1079; }","duration":"1.145155912s","start":"2026-03-26T14:52:42.126311Z","end":"2026-03-26T14:52:43.271467Z","steps":["trace[1137928152] 'agreement among raft nodes before linearized reading' (duration: 1.145082371s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:43.271505Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:42.126292Z","time spent":"1.145207534s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-26T14:52:43.275036Z","caller":"traceutil/trace.go:172","msg":"trace[369890313] transaction","detail":"{read_only:false; response_revision:1077; number_of_response:1; }","duration":"1.25018366s","start":"2026-03-26T14:52:42.024795Z","end":"2026-03-26T14:52:43.274978Z","steps":["trace[369890313] 'process raft request' (duration: 1.246232019s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:43.275260Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:42.024774Z","time spent":"1.250425214s","remote":"127.0.0.1:37578","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":1627,"response count":0,"response size":38,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-26T14:52:43.420459Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"363.724536ms"} {"level":"warn","ts":"2026-03-26T14:52:43.420541Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"5d78458d260361a5","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"363.814277ms"} {"level":"info","ts":"2026-03-26T14:52:43.480748Z","caller":"traceutil/trace.go:172","msg":"trace[1132223773] transaction","detail":"{read_only:false; response_revision:1080; number_of_response:1; }","duration":"543.324324ms","start":"2026-03-26T14:52:42.937394Z","end":"2026-03-26T14:52:43.480719Z","steps":["trace[1132223773] 'process raft request' (duration: 483.404943ms)","trace[1132223773] 'compare' (duration: 59.82596ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:43.481297Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:42.937362Z","time spent":"543.854901ms","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-26T14:52:43.772249Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367487564,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:52:44.261780Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"b59fd64328430f5b","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"641.216698ms"} {"level":"warn","ts":"2026-03-26T14:52:44.261865Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"5d78458d260361a5","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"641.30895ms"} {"level":"info","ts":"2026-03-26T14:52:44.261877Z","caller":"traceutil/trace.go:172","msg":"trace[351437700] linearizableReadLoop","detail":"{readStateIndex:1266; appliedIndex:1266; }","duration":"990.630127ms","start":"2026-03-26T14:52:43.271205Z","end":"2026-03-26T14:52:44.261835Z","steps":["trace[351437700] 'read index received' (duration: 990.618447ms)","trace[351437700] 'applied index is now lower than readState.Index' (duration: 10.03µs)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:44.762894Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367487567,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:52:45.264506Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367487567,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:52:45.305692Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.781578639s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:45.306388Z","caller":"traceutil/trace.go:172","msg":"trace[365144694] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1080; }","duration":"2.782243638s","start":"2026-03-26T14:52:42.524089Z","end":"2026-03-26T14:52:45.306333Z","steps":["trace[365144694] 'agreement among raft nodes before linearized reading' (duration: 1.737871948s)","trace[365144694] 'range keys from in-memory index tree' (duration: 1.043650491s)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:45.306600Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.257427601s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-26T14:52:45.306824Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:42.524066Z","time spent":"2.782748755s","remote":"127.0.0.1:36552","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:52:45.306526Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.925568463s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-03-26T14:52:45.306579Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.059803655s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:45.307011Z","caller":"traceutil/trace.go:172","msg":"trace[1869428470] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1080; }","duration":"2.06023142s","start":"2026-03-26T14:52:43.246770Z","end":"2026-03-26T14:52:45.307001Z","steps":["trace[1869428470] 'agreement among raft nodes before linearized reading' (duration: 1.015211591s)","trace[1869428470] 'range keys from in-memory index tree' (duration: 1.044583934s)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:45.307041Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.246748Z","time spent":"2.06028529s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:52:45.306729Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.044593064s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-26T14:52:45.307097Z","caller":"traceutil/trace.go:172","msg":"trace[787278577] transaction","detail":"{read_only:false; response_revision:1081; number_of_response:1; }","duration":"2.020459536s","start":"2026-03-26T14:52:43.286631Z","end":"2026-03-26T14:52:45.307090Z","steps":["trace[787278577] 'process raft request' (duration: 975.461297ms)","trace[787278577] 'compare' (duration: 1.044270829s)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:45.307128Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.286609Z","time spent":"2.020499877s","remote":"127.0.0.1:36910","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18511,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:52:45.307127Z","caller":"traceutil/trace.go:172","msg":"trace[676619106] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1080; }","duration":"2.926019729s","start":"2026-03-26T14:52:42.380934Z","end":"2026-03-26T14:52:45.306954Z","steps":["trace[676619106] 'agreement among raft nodes before linearized reading' (duration: 1.881037191s)","trace[676619106] 'range keys from in-memory index tree' (duration: 1.044443261s)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:45.307228Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:42.380913Z","time spent":"2.926282562s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":463,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-26T14:52:45.306921Z","caller":"traceutil/trace.go:172","msg":"trace[1243749682] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1080; }","duration":"1.257673854s","start":"2026-03-26T14:52:44.049152Z","end":"2026-03-26T14:52:45.306826Z","steps":["trace[1243749682] 'range keys from in-memory index tree' (duration: 1.257212986s)"],"step_count":1} {"level":"info","ts":"2026-03-26T14:52:45.625876Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 1275, vote: 5f170c75a1d97cc] ignored MsgPreVote from 5d78458d260361a5 [logterm: 2, index: 1271] at term 2: lease is not expired (remaining ticks: 7)"} {"level":"info","ts":"2026-03-26T14:52:45.627824Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"5f170c75a1d97cc [logterm: 2, index: 1275, vote: 5f170c75a1d97cc] ignored MsgVote from 5d78458d260361a5 [logterm: 2, index: 1271] at term 2: lease is not expired (remaining ticks: 7)"} {"level":"info","ts":"2026-03-26T14:52:45.629367Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"5f170c75a1d97cc [term: 2] received a MsgApp message with higher term from 5d78458d260361a5 [term: 3]"} {"level":"info","ts":"2026-03-26T14:52:45.629414Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"5f170c75a1d97cc became follower at term 3"} {"level":"info","ts":"2026-03-26T14:52:45.629431Z","logger":"raft","caller":"v3@v3.6.0/log.go:158","msg":"found conflict at index 1272 [existing term: 2, conflicting term: 3]"} {"level":"info","ts":"2026-03-26T14:52:45.629446Z","logger":"raft","caller":"v3@v3.6.0/log_unstable.go:211","msg":"truncate the unstable entries before index 1272"} {"level":"info","ts":"2026-03-26T14:52:45.629463Z","logger":"raft","caller":"v3@v3.6.0/node.go:372","msg":"raft.node: 5f170c75a1d97cc changed leader from 5f170c75a1d97cc to 5d78458d260361a5 at term 3"} {"level":"warn","ts":"2026-03-26T14:52:45.765678Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":10938290401367487567,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-26T14:52:45.924494Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.144469566s","expected-duration":"1s"} {"level":"info","ts":"2026-03-26T14:52:45.924616Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"5f170c75a1d97cc [term: 3] ignored a MsgAppResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:52:45.924651Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"5f170c75a1d97cc [term: 3] ignored a MsgAppResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:52:45.924668Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"5f170c75a1d97cc [term: 3] ignored a MsgAppResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:52:45.924708Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"5f170c75a1d97cc [term: 3] ignored a MsgAppResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:52:45.924726Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1164","msg":"5f170c75a1d97cc [term: 3] ignored entry appends from a MsgStorageAppendResp message with lower term [term: 2]"} {"level":"warn","ts":"2026-03-26T14:52:45.924881Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367487568,"received-request-id":10938290401367487564} {"level":"warn","ts":"2026-03-26T14:52:45.924991Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.636344481s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/jobs/envoy-gateway-system/envoy-gateway-gateway-helm-certgen\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-26T14:52:45.924938Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.510176259s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-26T14:52:45.924950Z","caller":"etcdserver/server.go:939","msg":"Failed to check current member's leadership","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-26T14:52:45.925046Z","caller":"traceutil/trace.go:172","msg":"trace[277448503] range","detail":"{range_begin:/registry/jobs/envoy-gateway-system/envoy-gateway-gateway-helm-certgen; range_end:; }","duration":"2.636425172s","start":"2026-03-26T14:52:43.288604Z","end":"2026-03-26T14:52:45.925029Z","steps":["trace[277448503] 'agreement among raft nodes before linearized reading' (duration: 2.636342091s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.925076Z","caller":"etcdserver/server.go:879","msg":"Ignore the lease revoking request because current member isn't a leader","local-member-id":428247441089599436} {"level":"info","ts":"2026-03-26T14:52:45.925075Z","caller":"traceutil/trace.go:172","msg":"trace[352027558] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; }","duration":"2.510342001s","start":"2026-03-26T14:52:43.414714Z","end":"2026-03-26T14:52:45.925056Z","steps":["trace[352027558] 'agreement among raft nodes before linearized reading' (duration: 2.510171828s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.925108Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.288584Z","time spent":"2.636509643s","remote":"127.0.0.1:37004","response type":"/etcdserverpb.KV/Range","request count":0,"request size":74,"response count":0,"response size":0,"request content":"key:\"/registry/jobs/envoy-gateway-system/envoy-gateway-gateway-helm-certgen\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:45.925129Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.648975497s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-26T14:52:45.925207Z","caller":"traceutil/trace.go:172","msg":"trace[676130546] range","detail":"{range_begin:; range_end:; }","duration":"2.649085658s","start":"2026-03-26T14:52:43.276105Z","end":"2026-03-26T14:52:45.925190Z","steps":["trace[676130546] 'agreement among raft nodes before linearized reading' (duration: 2.648970437s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.925232Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.414694Z","time spent":"2.510468192s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:45.925487Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.091122544s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-26T14:52:45.925579Z","caller":"traceutil/trace.go:172","msg":"trace[1322037073] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"2.091645311s","start":"2026-03-26T14:52:43.833917Z","end":"2026-03-26T14:52:45.925562Z","steps":["trace[1322037073] 'agreement among raft nodes before linearized reading' (duration: 2.091118194s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.925667Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.833896Z","time spent":"2.091756343s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"error","ts":"2026-03-26T14:52:45.972883Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[-]linearizable_read failed: etcdserver: leader changed\n[+]non_learner ok\n[+]data_corruption ok\n[+]serializable_read ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"info","ts":"2026-03-26T14:52:45.974075Z","caller":"traceutil/trace.go:172","msg":"trace[359714279] transaction","detail":"{read_only:false; response_revision:1082; number_of_response:1; }","duration":"2.357719975s","start":"2026-03-26T14:52:43.616332Z","end":"2026-03-26T14:52:45.974052Z","steps":["trace[359714279] 'process raft request' (duration: 2.308928805s)","trace[359714279] 'compare' (duration: 47.886957ms)"],"step_count":2} {"level":"warn","ts":"2026-03-26T14:52:45.974267Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:43.616274Z","time spent":"2.357927278s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-26T14:52:45.977652Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"5f170c75a1d97cc [term: 3] ignored a MsgAppResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:52:45.977713Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"5f170c75a1d97cc [term: 3] ignored a MsgAppResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:52:45.977728Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"5f170c75a1d97cc [term: 3] ignored a MsgAppResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:52:45.977739Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"5f170c75a1d97cc [term: 3] ignored a MsgAppResp message with lower term from 5f170c75a1d97cc [term: 2]"} {"level":"info","ts":"2026-03-26T14:52:45.979522Z","caller":"etcdserver/v3_server.go:903","msg":"first commit in current term: resending ReadIndex request"} {"level":"warn","ts":"2026-03-26T14:52:45.979925Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367487568,"received-request-id":10938290401367487567} {"level":"warn","ts":"2026-03-26T14:52:45.989441Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":10938290401367487569,"received-request-id":10938290401367487568} {"level":"warn","ts":"2026-03-26T14:52:45.989542Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.460171221s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:45.989588Z","caller":"traceutil/trace.go:172","msg":"trace[1286360060] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1099; }","duration":"1.460207711s","start":"2026-03-26T14:52:44.529351Z","end":"2026-03-26T14:52:45.989559Z","steps":["trace[1286360060] 'agreement among raft nodes before linearized reading' (duration: 1.460155651s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.989619Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.529330Z","time spent":"1.460284213s","remote":"127.0.0.1:36562","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":27,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-26T14:52:45.989660Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.072324966s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:510"} {"level":"info","ts":"2026-03-26T14:52:45.989744Z","caller":"traceutil/trace.go:172","msg":"trace[881788398] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1099; }","duration":"1.072382776s","start":"2026-03-26T14:52:44.917318Z","end":"2026-03-26T14:52:45.989701Z","steps":["trace[881788398] 'agreement among raft nodes before linearized reading' (duration: 1.072140783s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.989785Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.917297Z","time spent":"1.072477807s","remote":"127.0.0.1:37064","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:45.989883Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.464273265s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-26T14:52:45.989934Z","caller":"traceutil/trace.go:172","msg":"trace[260150407] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649; range_end:; response_count:1; response_revision:1099; }","duration":"1.464330986s","start":"2026-03-26T14:52:44.525591Z","end":"2026-03-26T14:52:45.989922Z","steps":["trace[260150407] 'agreement among raft nodes before linearized reading' (duration: 1.464182714s)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.989987Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:44.525572Z","time spent":"1.464392037s","remote":"127.0.0.1:36678","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":816,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.18a06bb3f85ba649\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:52:45.989655Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"371.212942ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ipaddresses\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-26T14:52:45.990832Z","caller":"traceutil/trace.go:172","msg":"trace[1472336708] range","detail":"{range_begin:/registry/ipaddresses; range_end:; response_count:0; response_revision:1099; }","duration":"372.386658ms","start":"2026-03-26T14:52:45.618428Z","end":"2026-03-26T14:52:45.990815Z","steps":["trace[1472336708] 'agreement among raft nodes before linearized reading' (duration: 371.191021ms)"],"step_count":1} {"level":"warn","ts":"2026-03-26T14:52:45.991164Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-26T14:52:45.618406Z","time spent":"372.461468ms","remote":"127.0.0.1:37146","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":27,"request content":"key:\"/registry/ipaddresses\" limit:1 "} {"level":"warn","ts":"2026-03-26T14:53:09.743086Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36040","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-26T14:53:09.753992Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:36062","server-name":"","error":"EOF"}