{"level":"warn","ts":"2026-03-19T21:51:45.19888Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-19T21:51:45.198969Z","caller":"etcdmain/etcd.go:73","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.19.213.66:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--experimental-initial-corrupt-check=true","--experimental-watch-progress-notify-interval=5s","--initial-advertise-peer-urls=https://199.19.213.66:2380","--initial-cluster=instance=https://199.19.213.66:2380","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.19.213.66:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.19.213.66:2380","--name=instance","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt"]} {"level":"warn","ts":"2026-03-19T21:51:45.19904Z","caller":"embed/config.go:679","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-19T21:51:45.199051Z","caller":"embed/etcd.go:127","msg":"configuring peer listeners","listen-peer-urls":["https://199.19.213.66:2380"]} {"level":"info","ts":"2026-03-19T21:51:45.19907Z","caller":"embed/etcd.go:494","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-19T21:51:45.199701Z","caller":"embed/etcd.go:135","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.66:2379"]} {"level":"info","ts":"2026-03-19T21:51:45.199776Z","caller":"embed/etcd.go:308","msg":"starting an etcd server","etcd-version":"3.5.12","git-sha":"e7b3bb6cc","go-version":"go1.20.13","go-os":"linux","go-arch":"amd64","max-cpu-set":16,"max-cpu-available":16,"member-initialized":false,"name":"instance","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.19.213.66:2380"],"listen-peer-urls":["https://199.19.213.66:2380"],"advertise-client-urls":["https://199.19.213.66:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.66:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"instance=https://199.19.213.66:2380","initial-cluster-state":"new","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"} {"level":"info","ts":"2026-03-19T21:51:45.216848Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"14.786034ms"} {"level":"info","ts":"2026-03-19T21:51:45.233992Z","caller":"etcdserver/raft.go:495","msg":"starting local member","local-member-id":"dc35f804a62878ec","cluster-id":"ef5544f2dd66d370"} {"level":"info","ts":"2026-03-19T21:51:45.234125Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec switched to configuration voters=()"} {"level":"info","ts":"2026-03-19T21:51:45.234156Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec became follower at term 0"} {"level":"info","ts":"2026-03-19T21:51:45.234176Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"newRaft dc35f804a62878ec [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"info","ts":"2026-03-19T21:51:45.234194Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec became follower at term 1"} {"level":"info","ts":"2026-03-19T21:51:45.234247Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec switched to configuration voters=(15867861560961038572)"} {"level":"warn","ts":"2026-03-19T21:51:45.248233Z","caller":"auth/store.go:1241","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-19T21:51:45.252261Z","caller":"mvcc/kvstore.go:407","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-19T21:51:45.255105Z","caller":"etcdserver/quota.go:94","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-19T21:51:45.259039Z","caller":"etcdserver/server.go:860","msg":"starting etcd server","local-member-id":"dc35f804a62878ec","local-server-version":"3.5.12","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-19T21:51:45.259142Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-19T21:51:45.259252Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-19T21:51:45.259265Z","caller":"fileutil/purge.go:50","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-19T21:51:45.25924Z","caller":"etcdserver/server.go:744","msg":"started as single-node; fast-forwarding election ticks","local-member-id":"dc35f804a62878ec","forward-ticks":9,"forward-duration":"900ms","election-ticks":10,"election-timeout":"1s"} {"level":"info","ts":"2026-03-19T21:51:45.260521Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec switched to configuration voters=(15867861560961038572)"} {"level":"info","ts":"2026-03-19T21:51:45.260658Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"ef5544f2dd66d370","local-member-id":"dc35f804a62878ec","added-peer-id":"dc35f804a62878ec","added-peer-peer-urls":["https://199.19.213.66:2380"]} {"level":"info","ts":"2026-03-19T21:51:45.261764Z","caller":"embed/etcd.go:726","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-19T21:51:45.26185Z","caller":"embed/etcd.go:597","msg":"serving peer traffic","address":"199.19.213.66:2380"} {"level":"info","ts":"2026-03-19T21:51:45.26193Z","caller":"embed/etcd.go:569","msg":"cmux::serve","address":"199.19.213.66:2380"} {"level":"info","ts":"2026-03-19T21:51:45.262026Z","caller":"embed/etcd.go:277","msg":"now serving peer/client/metrics","local-member-id":"dc35f804a62878ec","initial-advertise-peer-urls":["https://199.19.213.66:2380"],"listen-peer-urls":["https://199.19.213.66:2380"],"advertise-client-urls":["https://199.19.213.66:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.66:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-19T21:51:45.262081Z","caller":"embed/etcd.go:857","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-19T21:51:45.535085Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec is starting a new election at term 1"} {"level":"info","ts":"2026-03-19T21:51:45.535246Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec became pre-candidate at term 1"} {"level":"info","ts":"2026-03-19T21:51:45.535291Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec received MsgPreVoteResp from dc35f804a62878ec at term 1"} {"level":"info","ts":"2026-03-19T21:51:45.535342Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec became candidate at term 2"} {"level":"info","ts":"2026-03-19T21:51:45.535361Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec received MsgVoteResp from dc35f804a62878ec at term 2"} {"level":"info","ts":"2026-03-19T21:51:45.535377Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"dc35f804a62878ec became leader at term 2"} {"level":"info","ts":"2026-03-19T21:51:45.535422Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: dc35f804a62878ec elected leader dc35f804a62878ec at term 2"} {"level":"info","ts":"2026-03-19T21:51:45.536454Z","caller":"etcdserver/server.go:2578","msg":"setting up initial cluster version using v2 API","cluster-version":"3.5"} {"level":"info","ts":"2026-03-19T21:51:45.536665Z","caller":"etcdserver/server.go:2068","msg":"published local member to cluster through raft","local-member-id":"dc35f804a62878ec","local-member-attributes":"{Name:instance ClientURLs:[https://199.19.213.66:2379]}","request-path":"/0/members/dc35f804a62878ec/attributes","cluster-id":"ef5544f2dd66d370","publish-timeout":"7s"} {"level":"info","ts":"2026-03-19T21:51:45.536699Z","caller":"embed/serve.go:103","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-19T21:51:45.536739Z","caller":"embed/serve.go:103","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-19T21:51:45.536979Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-19T21:51:45.537024Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-03-19T21:51:45.538774Z","caller":"membership/cluster.go:584","msg":"set initial cluster version","cluster-id":"ef5544f2dd66d370","local-member-id":"dc35f804a62878ec","cluster-version":"3.5"} {"level":"info","ts":"2026-03-19T21:51:45.538961Z","caller":"api/capability.go:75","msg":"enabled capabilities for version","cluster-version":"3.5"} {"level":"info","ts":"2026-03-19T21:51:45.539066Z","caller":"etcdserver/server.go:2602","msg":"cluster version is updated","cluster-version":"3.5"} {"level":"info","ts":"2026-03-19T21:51:45.53946Z","caller":"embed/serve.go:250","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.19.213.66:2379"} {"level":"info","ts":"2026-03-19T21:51:45.540213Z","caller":"embed/serve.go:250","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"info","ts":"2026-03-19T21:52:23.267057Z","caller":"traceutil/trace.go:171","msg":"trace[1270974411] transaction","detail":"{read_only:false; response_revision:551; number_of_response:1; }","duration":"213.821674ms","start":"2026-03-19T21:52:23.053177Z","end":"2026-03-19T21:52:23.266999Z","steps":["trace[1270974411] 'process raft request' (duration: 213.515296ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:53:34.286955Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"214.937568ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" ","response":"range_response_count:1 size:554"} {"level":"info","ts":"2026-03-19T21:53:34.287044Z","caller":"traceutil/trace.go:171","msg":"trace[1389981595] range","detail":"{range_begin:/registry/leases/openstack/rabbitmq-cluster-operator-leader-election; range_end:; response_count:1; response_revision:1590; }","duration":"215.079272ms","start":"2026-03-19T21:53:34.07195Z","end":"2026-03-19T21:53:34.287029Z","steps":["trace[1389981595] 'range keys from in-memory index tree' (duration: 214.771353ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:53:34.814788Z","caller":"traceutil/trace.go:171","msg":"trace[2022241262] transaction","detail":"{read_only:false; response_revision:1595; number_of_response:1; }","duration":"169.274727ms","start":"2026-03-19T21:53:34.64548Z","end":"2026-03-19T21:53:34.814755Z","steps":["trace[2022241262] 'process raft request' (duration: 169.01936ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:12.301107Z","caller":"traceutil/trace.go:171","msg":"trace[2043331792] transaction","detail":"{read_only:false; response_revision:1828; number_of_response:1; }","duration":"137.860345ms","start":"2026-03-19T21:54:12.163188Z","end":"2026-03-19T21:54:12.301048Z","steps":["trace[2043331792] 'process raft request' (duration: 137.446442ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:12.325069Z","caller":"traceutil/trace.go:171","msg":"trace[1319164321] transaction","detail":"{read_only:false; response_revision:1829; number_of_response:1; }","duration":"157.464801ms","start":"2026-03-19T21:54:12.167555Z","end":"2026-03-19T21:54:12.32502Z","steps":["trace[1319164321] 'process raft request' (duration: 157.206474ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:12.751019Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"110.025961ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:54:12.751183Z","caller":"traceutil/trace.go:171","msg":"trace[1036984601] transaction","detail":"{read_only:false; response_revision:1831; number_of_response:1; }","duration":"155.346951ms","start":"2026-03-19T21:54:12.595786Z","end":"2026-03-19T21:54:12.751133Z","steps":["trace[1036984601] 'process raft request' (duration: 44.63602ms)","trace[1036984601] 'compare' (duration: 109.895598ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:54:13.774409Z","caller":"traceutil/trace.go:171","msg":"trace[203377900] transaction","detail":"{read_only:false; response_revision:1838; number_of_response:1; }","duration":"106.334077ms","start":"2026-03-19T21:54:13.668058Z","end":"2026-03-19T21:54:13.774392Z","steps":["trace[203377900] 'process raft request' (duration: 106.235874ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:15.972241Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.719686513s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:54:15.97232Z","caller":"traceutil/trace.go:171","msg":"trace[2031395666] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1838; }","duration":"1.719815276s","start":"2026-03-19T21:54:14.252486Z","end":"2026-03-19T21:54:15.972301Z","steps":["trace[2031395666] 'range keys from in-memory index tree' (duration: 1.719521047s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:15.972357Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:14.252466Z","time spent":"1.719882909s","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":29,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-19T21:54:15.972241Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.035951924s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" ","response":"range_response_count:1 size:5444"} {"level":"info","ts":"2026-03-19T21:54:15.972506Z","caller":"traceutil/trace.go:171","msg":"trace[386300406] range","detail":"{range_begin:/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb; range_end:; response_count:1; response_revision:1838; }","duration":"2.036261593s","start":"2026-03-19T21:54:13.936228Z","end":"2026-03-19T21:54:15.972489Z","steps":["trace[386300406] 'range keys from in-memory index tree' (duration: 2.035766218s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:15.972485Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.807876051s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" ","response":"range_response_count:1 size:505"} {"level":"warn","ts":"2026-03-19T21:54:15.972544Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:13.936194Z","time spent":"2.036340235s","remote":"127.0.0.1:51270","response type":"/etcdserverpb.KV/Range","request count":0,"request size":74,"response count":1,"response size":5468,"request content":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" "} {"level":"info","ts":"2026-03-19T21:54:15.972575Z","caller":"traceutil/trace.go:171","msg":"trace[1729036983] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1838; }","duration":"1.808015676s","start":"2026-03-19T21:54:14.164535Z","end":"2026-03-19T21:54:15.972551Z","steps":["trace[1729036983] 'range keys from in-memory index tree' (duration: 1.807666956s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:15.97263Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:14.164514Z","time spent":"1.808103368s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":54,"response count":1,"response size":529,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" "} {"level":"warn","ts":"2026-03-19T21:54:16.977629Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"233.154436ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:54:16.977745Z","caller":"traceutil/trace.go:171","msg":"trace[1173448355] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1838; }","duration":"233.279059ms","start":"2026-03-19T21:54:16.744449Z","end":"2026-03-19T21:54:16.977728Z","steps":["trace[1173448355] 'range keys from in-memory index tree' (duration: 233.140075ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.977979Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000264998s","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:29"} {"level":"info","ts":"2026-03-19T21:54:16.979275Z","caller":"traceutil/trace.go:171","msg":"trace[1208569068] transaction","detail":"{read_only:false; response_revision:1839; number_of_response:1; }","duration":"2.40371934s","start":"2026-03-19T21:54:14.575266Z","end":"2026-03-19T21:54:16.978985Z","steps":["trace[1208569068] 'process raft request' (duration: 2.40300245s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.979926Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:14.575241Z","time spent":"2.404112242s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":555,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:16.98001Z","caller":"traceutil/trace.go:171","msg":"trace[1898473258] linearizableReadLoop","detail":"{readStateIndex:1911; appliedIndex:1909; }","duration":"2.224868229s","start":"2026-03-19T21:54:14.755089Z","end":"2026-03-19T21:54:16.979957Z","steps":["trace[1898473258] 'read index received' (duration: 222.497812ms)","trace[1898473258] 'applied index is now lower than readState.Index' (duration: 2.002313396s)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:54:16.980461Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.225369854s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" ","response":"range_response_count:1 size:479"} {"level":"info","ts":"2026-03-19T21:54:16.980535Z","caller":"traceutil/trace.go:171","msg":"trace[1921633331] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1839; }","duration":"2.225468597s","start":"2026-03-19T21:54:14.755038Z","end":"2026-03-19T21:54:16.980506Z","steps":["trace[1921633331] 'agreement among raft nodes before linearized reading' (duration: 2.225058445s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.980587Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:14.755018Z","time spent":"2.225560189s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":45,"response count":1,"response size":503,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" "} {"level":"info","ts":"2026-03-19T21:54:16.98075Z","caller":"traceutil/trace.go:171","msg":"trace[1840672290] transaction","detail":"{read_only:false; response_revision:1840; number_of_response:1; }","duration":"1.467867506s","start":"2026-03-19T21:54:15.512852Z","end":"2026-03-19T21:54:16.98072Z","steps":["trace[1840672290] 'process raft request' (duration: 1.466985571s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.980935Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:15.512832Z","time spent":"1.467984959s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":509,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:16.982164Z","caller":"traceutil/trace.go:171","msg":"trace[1498881121] transaction","detail":"{read_only:false; response_revision:1841; number_of_response:1; }","duration":"1.005122576s","start":"2026-03-19T21:54:15.977024Z","end":"2026-03-19T21:54:16.982147Z","steps":["trace[1498881121] 'process raft request' (duration: 1.003488389s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.982363Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:15.977003Z","time spent":"1.005185698s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":490,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:16.982518Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.173356184s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/\" range_end:\"/registry/configmaps0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-03-19T21:54:16.982619Z","caller":"traceutil/trace.go:171","msg":"trace[694081116] range","detail":"{range_begin:/registry/configmaps/; range_end:/registry/configmaps0; response_count:0; response_revision:1841; }","duration":"2.173482207s","start":"2026-03-19T21:54:14.809109Z","end":"2026-03-19T21:54:16.982591Z","steps":["trace[694081116] 'agreement among raft nodes before linearized reading' (duration: 2.173332873s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.982689Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:14.809088Z","time spent":"2.173586201s","remote":"127.0.0.1:56962","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":23,"response size":31,"request content":"key:\"/registry/configmaps/\" range_end:\"/registry/configmaps0\" count_only:true "} {"level":"warn","ts":"2026-03-19T21:54:16.982942Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"880.246881ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cert-manager.io/clusterissuers/\" range_end:\"/registry/cert-manager.io/clusterissuers0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-03-19T21:54:16.983035Z","caller":"traceutil/trace.go:171","msg":"trace[1171361977] range","detail":"{range_begin:/registry/cert-manager.io/clusterissuers/; range_end:/registry/cert-manager.io/clusterissuers0; response_count:0; response_revision:1841; }","duration":"880.362645ms","start":"2026-03-19T21:54:16.102644Z","end":"2026-03-19T21:54:16.983006Z","steps":["trace[1171361977] 'agreement among raft nodes before linearized reading' (duration: 880.260782ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.983104Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:16.102623Z","time spent":"880.464087ms","remote":"127.0.0.1:52168","response type":"/etcdserverpb.KV/Range","request count":0,"request size":88,"response count":2,"response size":31,"request content":"key:\"/registry/cert-manager.io/clusterissuers/\" range_end:\"/registry/cert-manager.io/clusterissuers0\" count_only:true "} {"level":"warn","ts":"2026-03-19T21:54:16.9827Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.17357659s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" ","response":"range_response_count:1 size:555"} {"level":"warn","ts":"2026-03-19T21:54:16.983225Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.677030298s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" ","response":"range_response_count:1 size:492"} {"level":"warn","ts":"2026-03-19T21:54:16.983367Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.658064609s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-19T21:54:16.983362Z","caller":"traceutil/trace.go:171","msg":"trace[1720179717] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:1841; }","duration":"1.677174513s","start":"2026-03-19T21:54:15.306137Z","end":"2026-03-19T21:54:16.983311Z","steps":["trace[1720179717] 'agreement among raft nodes before linearized reading' (duration: 1.676818343s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:16.983235Z","caller":"traceutil/trace.go:171","msg":"trace[129156332] range","detail":"{range_begin:/registry/leases/openstack/rabbitmq-cluster-operator-leader-election; range_end:; response_count:1; response_revision:1841; }","duration":"2.1742709s","start":"2026-03-19T21:54:14.808952Z","end":"2026-03-19T21:54:16.983222Z","steps":["trace[129156332] 'agreement among raft nodes before linearized reading' (duration: 2.17356841s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:16.983459Z","caller":"traceutil/trace.go:171","msg":"trace[2133714476] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1841; }","duration":"1.658154482s","start":"2026-03-19T21:54:15.325264Z","end":"2026-03-19T21:54:16.983418Z","steps":["trace[2133714476] 'agreement among raft nodes before linearized reading' (duration: 1.657973227s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.983463Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:15.306121Z","time spent":"1.677312706s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":60,"response count":1,"response size":516,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" "} {"level":"warn","ts":"2026-03-19T21:54:16.983511Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:15.325253Z","time spent":"1.658247604s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" "} {"level":"warn","ts":"2026-03-19T21:54:16.983498Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.704231922s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:432"} {"level":"warn","ts":"2026-03-19T21:54:16.983496Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:14.808928Z","time spent":"2.174556167s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":70,"response count":1,"response size":579,"request content":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" "} {"level":"warn","ts":"2026-03-19T21:54:16.983369Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.009230042s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:54:16.983845Z","caller":"traceutil/trace.go:171","msg":"trace[456990676] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1841; }","duration":"1.009712376s","start":"2026-03-19T21:54:15.974095Z","end":"2026-03-19T21:54:16.983807Z","steps":["trace[456990676] 'agreement among raft nodes before linearized reading' (duration: 1.009140709s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:16.983877Z","caller":"traceutil/trace.go:171","msg":"trace[1963872721] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1841; }","duration":"1.70451646s","start":"2026-03-19T21:54:15.279213Z","end":"2026-03-19T21:54:16.98373Z","steps":["trace[1963872721] 'agreement among raft nodes before linearized reading' (duration: 1.704025487s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:16.983915Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:15.974053Z","time spent":"1.00984825s","remote":"127.0.0.1:56892","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":29,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-19T21:54:16.98395Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:15.279188Z","time spent":"1.704733707s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":44,"response count":1,"response size":456,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" "} {"level":"warn","ts":"2026-03-19T21:54:17.893668Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"768.180452ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:54:17.893952Z","caller":"traceutil/trace.go:171","msg":"trace[1938762104] transaction","detail":"{read_only:false; response_revision:1842; number_of_response:1; }","duration":"907.007084ms","start":"2026-03-19T21:54:16.986855Z","end":"2026-03-19T21:54:17.893862Z","steps":["trace[1938762104] 'process raft request' (duration: 138.547674ms)","trace[1938762104] 'compare' (duration: 767.949866ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:54:17.89406Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"902.460233ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:54:17.8941Z","caller":"traceutil/trace.go:171","msg":"trace[37861115] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1846; }","duration":"902.516996ms","start":"2026-03-19T21:54:16.991574Z","end":"2026-03-19T21:54:17.89409Z","steps":["trace[37861115] 'agreement among raft nodes before linearized reading' (duration: 902.452084ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:17.894085Z","caller":"traceutil/trace.go:171","msg":"trace[600230273] transaction","detail":"{read_only:false; response_revision:1844; number_of_response:1; }","duration":"905.996575ms","start":"2026-03-19T21:54:16.988056Z","end":"2026-03-19T21:54:17.894052Z","steps":["trace[600230273] 'process raft request' (duration: 905.792759ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:17.894139Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:16.991566Z","time spent":"902.565207ms","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":29,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-19T21:54:17.894141Z","caller":"traceutil/trace.go:171","msg":"trace[2027304619] transaction","detail":"{read_only:false; response_revision:1845; number_of_response:1; }","duration":"905.639724ms","start":"2026-03-19T21:54:16.988461Z","end":"2026-03-19T21:54:17.894101Z","steps":["trace[2027304619] 'process raft request' (duration: 905.418548ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:17.894149Z","caller":"traceutil/trace.go:171","msg":"trace[700531400] transaction","detail":"{read_only:false; response_revision:1846; number_of_response:1; }","duration":"904.917454ms","start":"2026-03-19T21:54:16.989194Z","end":"2026-03-19T21:54:17.894112Z","steps":["trace[700531400] 'process raft request' (duration: 904.749799ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:17.894201Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:16.988038Z","time spent":"906.103857ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":540,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:17.894155Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:16.986834Z","time spent":"907.215369ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":464,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:17.894296Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:16.98845Z","time spent":"905.771508ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":418,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:17.894318Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:16.989178Z","time spent":"905.057698ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":477,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:17.893997Z","caller":"traceutil/trace.go:171","msg":"trace[546562930] linearizableReadLoop","detail":"{readStateIndex:1918; appliedIndex:1913; }","duration":"902.385091ms","start":"2026-03-19T21:54:16.991594Z","end":"2026-03-19T21:54:17.893979Z","steps":["trace[546562930] 'read index received' (duration: 133.822869ms)","trace[546562930] 'applied index is now lower than readState.Index' (duration: 768.561132ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:54:17.894452Z","caller":"traceutil/trace.go:171","msg":"trace[1712457772] transaction","detail":"{read_only:false; response_revision:1843; number_of_response:1; }","duration":"906.942812ms","start":"2026-03-19T21:54:16.987346Z","end":"2026-03-19T21:54:17.894289Z","steps":["trace[1712457772] 'process raft request' (duration: 906.449538ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:17.894627Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"764.357054ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/openstack/percona-xtradb-haproxy-0.189e5cb680114d93\" ","response":"range_response_count:1 size:846"} {"level":"info","ts":"2026-03-19T21:54:17.894699Z","caller":"traceutil/trace.go:171","msg":"trace[1247374039] range","detail":"{range_begin:/registry/events/openstack/percona-xtradb-haproxy-0.189e5cb680114d93; range_end:; response_count:1; response_revision:1846; }","duration":"764.480327ms","start":"2026-03-19T21:54:17.130198Z","end":"2026-03-19T21:54:17.894678Z","steps":["trace[1247374039] 'agreement among raft nodes before linearized reading' (duration: 764.269061ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:17.894743Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:17.130103Z","time spent":"764.630672ms","remote":"127.0.0.1:56936","response type":"/etcdserverpb.KV/Range","request count":0,"request size":70,"response count":1,"response size":870,"request content":"key:\"/registry/events/openstack/percona-xtradb-haproxy-0.189e5cb680114d93\" "} {"level":"warn","ts":"2026-03-19T21:54:17.894862Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:16.987332Z","time spent":"907.272651ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":422,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:19.945264Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.842547909s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:54:19.945386Z","caller":"traceutil/trace.go:171","msg":"trace[314602063] linearizableReadLoop","detail":"{readStateIndex:1919; appliedIndex:1918; }","duration":"1.984833768s","start":"2026-03-19T21:54:17.96053Z","end":"2026-03-19T21:54:19.945364Z","steps":["trace[314602063] 'read index received' (duration: 141.9959ms)","trace[314602063] 'applied index is now lower than readState.Index' (duration: 1.842835548s)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:54:19.945449Z","caller":"traceutil/trace.go:171","msg":"trace[578059891] transaction","detail":"{read_only:false; response_revision:1847; number_of_response:1; }","duration":"2.047735918s","start":"2026-03-19T21:54:17.89765Z","end":"2026-03-19T21:54:19.945386Z","steps":["trace[578059891] 'process raft request' (duration: 204.8783ms)","trace[578059891] 'compare' (duration: 1.842070226s)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:54:19.945625Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.985071374s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" ","response":"range_response_count:1 size:12385"} {"level":"info","ts":"2026-03-19T21:54:19.94567Z","caller":"traceutil/trace.go:171","msg":"trace[577083266] range","detail":"{range_begin:/registry/pods/openstack/percona-xtradb-pxc-0; range_end:; response_count:1; response_revision:1847; }","duration":"1.985167687s","start":"2026-03-19T21:54:17.960489Z","end":"2026-03-19T21:54:19.945656Z","steps":["trace[577083266] 'agreement among raft nodes before linearized reading' (duration: 1.984971571s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:19.945722Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:17.96047Z","time spent":"1.985240329s","remote":"127.0.0.1:57034","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":12409,"request content":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" "} {"level":"warn","ts":"2026-03-19T21:54:19.946269Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:17.897629Z","time spent":"2.048017967s","remote":"127.0.0.1:56936","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":831,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:19.955998Z","caller":"traceutil/trace.go:171","msg":"trace[18856391] transaction","detail":"{read_only:false; response_revision:1848; number_of_response:1; }","duration":"969.722128ms","start":"2026-03-19T21:54:18.986257Z","end":"2026-03-19T21:54:19.95598Z","steps":["trace[18856391] 'process raft request' (duration: 969.623355ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:19.956125Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:18.986215Z","time spent":"969.851281ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":509,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:20.251826Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000246967s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-19T21:54:20.251965Z","caller":"traceutil/trace.go:171","msg":"trace[1342561452] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000450862s","start":"2026-03-19T21:54:18.251489Z","end":"2026-03-19T21:54:20.251939Z","steps":["trace[1342561452] 'agreement among raft nodes before linearized reading' (duration: 2.000294548s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:20.252032Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:18.251458Z","time spent":"2.000559846s","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/19 21:54:20 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-19T21:54:21.193348Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"1.237453578s","expected-duration":"1s"} {"level":"info","ts":"2026-03-19T21:54:21.193964Z","caller":"traceutil/trace.go:171","msg":"trace[134798942] linearizableReadLoop","detail":"{readStateIndex:1921; appliedIndex:1919; }","duration":"1.248465122s","start":"2026-03-19T21:54:19.945427Z","end":"2026-03-19T21:54:21.193892Z","steps":["trace[134798942] 'read index received' (duration: 10.462549ms)","trace[134798942] 'applied index is now lower than readState.Index' (duration: 1.238001343s)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:54:21.194209Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.795592473s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-19T21:54:21.194249Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.196772839s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.19.213.66\" ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-19T21:54:21.194265Z","caller":"traceutil/trace.go:171","msg":"trace[73101045] range","detail":"{range_begin:/registry/horizontalpodautoscalers/; range_end:/registry/horizontalpodautoscalers0; response_count:0; response_revision:1849; }","duration":"2.795678355s","start":"2026-03-19T21:54:18.39857Z","end":"2026-03-19T21:54:21.194248Z","steps":["trace[73101045] 'agreement among raft nodes before linearized reading' (duration: 2.795518881s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.194287Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.294745398s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:433"} {"level":"warn","ts":"2026-03-19T21:54:21.194316Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:18.398515Z","time spent":"2.795791179s","remote":"127.0.0.1:57066","response type":"/etcdserverpb.KV/Range","request count":0,"request size":76,"response count":0,"response size":29,"request content":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true "} {"level":"info","ts":"2026-03-19T21:54:21.194305Z","caller":"traceutil/trace.go:171","msg":"trace[1619235838] range","detail":"{range_begin:/registry/masterleases/199.19.213.66; range_end:; response_count:1; response_revision:1849; }","duration":"2.196888373s","start":"2026-03-19T21:54:18.997402Z","end":"2026-03-19T21:54:21.19429Z","steps":["trace[1619235838] 'agreement among raft nodes before linearized reading' (duration: 2.196631156s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:21.194372Z","caller":"traceutil/trace.go:171","msg":"trace[1950186809] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1849; }","duration":"1.294874252s","start":"2026-03-19T21:54:19.899469Z","end":"2026-03-19T21:54:21.194343Z","steps":["trace[1950186809] 'agreement among raft nodes before linearized reading' (duration: 1.294594654s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.194401Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:19.899448Z","time spent":"1.294946484s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":44,"response count":1,"response size":457,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" "} {"level":"warn","ts":"2026-03-19T21:54:21.194466Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:18.997388Z","time spent":"2.197004276s","remote":"127.0.0.1:56898","response type":"/etcdserverpb.KV/Range","request count":0,"request size":38,"response count":1,"response size":157,"request content":"key:\"/registry/masterleases/199.19.213.66\" "} {"level":"warn","ts":"2026-03-19T21:54:21.194435Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.6528313s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumclusterwidenetworkpolicies/\" range_end:\"/registry/cilium.io/ciliumclusterwidenetworkpolicies0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-19T21:54:21.194529Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.294754409s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-19T21:54:21.194565Z","caller":"traceutil/trace.go:171","msg":"trace[1814755754] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1849; }","duration":"1.29480779s","start":"2026-03-19T21:54:19.899746Z","end":"2026-03-19T21:54:21.194554Z","steps":["trace[1814755754] 'agreement among raft nodes before linearized reading' (duration: 1.294640665s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:21.194564Z","caller":"traceutil/trace.go:171","msg":"trace[228056554] range","detail":"{range_begin:/registry/cilium.io/ciliumclusterwidenetworkpolicies/; range_end:/registry/cilium.io/ciliumclusterwidenetworkpolicies0; response_count:0; response_revision:1849; }","duration":"1.653021485s","start":"2026-03-19T21:54:19.541511Z","end":"2026-03-19T21:54:21.194532Z","steps":["trace[228056554] 'agreement among raft nodes before linearized reading' (duration: 1.652805309s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:21.194581Z","caller":"traceutil/trace.go:171","msg":"trace[76677293] transaction","detail":"{read_only:false; response_revision:1849; number_of_response:1; }","duration":"2.207635079s","start":"2026-03-19T21:54:18.986879Z","end":"2026-03-19T21:54:21.194515Z","steps":["trace[76677293] 'process raft request' (duration: 2.206793806s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.194678Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.2944391s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" ","response":"range_response_count:1 size:555"} {"level":"info","ts":"2026-03-19T21:54:21.194718Z","caller":"traceutil/trace.go:171","msg":"trace[1747708514] range","detail":"{range_begin:/registry/leases/openstack/rabbitmq-cluster-operator-leader-election; range_end:; response_count:1; response_revision:1849; }","duration":"1.294513652s","start":"2026-03-19T21:54:19.900196Z","end":"2026-03-19T21:54:21.19471Z","steps":["trace[1747708514] 'agreement among raft nodes before linearized reading' (duration: 1.294147802s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.194679Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.204723347s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" ","response":"range_response_count:1 size:5444"} {"level":"warn","ts":"2026-03-19T21:54:21.194677Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.206687252s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" ","response":"range_response_count:1 size:505"} {"level":"warn","ts":"2026-03-19T21:54:21.194649Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:19.54149Z","time spent":"1.653147738s","remote":"127.0.0.1:52182","response type":"/etcdserverpb.KV/Range","request count":0,"request size":112,"response count":0,"response size":29,"request content":"key:\"/registry/cilium.io/ciliumclusterwidenetworkpolicies/\" range_end:\"/registry/cilium.io/ciliumclusterwidenetworkpolicies0\" count_only:true "} {"level":"info","ts":"2026-03-19T21:54:21.194786Z","caller":"traceutil/trace.go:171","msg":"trace[425240101] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1849; }","duration":"2.206843836s","start":"2026-03-19T21:54:18.987936Z","end":"2026-03-19T21:54:21.19478Z","steps":["trace[425240101] 'agreement among raft nodes before linearized reading' (duration: 2.206115745s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.194804Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:18.987914Z","time spent":"2.206885257s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":54,"response count":1,"response size":529,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" "} {"level":"info","ts":"2026-03-19T21:54:21.194818Z","caller":"traceutil/trace.go:171","msg":"trace[2014420845] range","detail":"{range_begin:/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb; range_end:; response_count:1; response_revision:1849; }","duration":"3.204898521s","start":"2026-03-19T21:54:17.989881Z","end":"2026-03-19T21:54:21.194779Z","steps":["trace[2014420845] 'agreement among raft nodes before linearized reading' (duration: 3.204406627s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.194586Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:19.899735Z","time spent":"1.294846231s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":461,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" "} {"level":"warn","ts":"2026-03-19T21:54:21.194747Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:19.900174Z","time spent":"1.294563853s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":70,"response count":1,"response size":579,"request content":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" "} {"level":"warn","ts":"2026-03-19T21:54:21.19485Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:18.986842Z","time spent":"2.207885516s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":555,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:21.194888Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:17.98987Z","time spent":"3.205007455s","remote":"127.0.0.1:51270","response type":"/etcdserverpb.KV/Range","request count":0,"request size":74,"response count":1,"response size":5468,"request content":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" "} {"level":"warn","ts":"2026-03-19T21:54:21.194975Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.295073077s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" ","response":"range_response_count:1 size:479"} {"level":"info","ts":"2026-03-19T21:54:21.195143Z","caller":"traceutil/trace.go:171","msg":"trace[1538593179] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1849; }","duration":"1.295419417s","start":"2026-03-19T21:54:19.899685Z","end":"2026-03-19T21:54:21.195105Z","steps":["trace[1538593179] 'agreement among raft nodes before linearized reading' (duration: 1.294270905s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.195248Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.295476559s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" ","response":"range_response_count:1 size:492"} {"level":"info","ts":"2026-03-19T21:54:21.195289Z","caller":"traceutil/trace.go:171","msg":"trace[1324946703] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:1849; }","duration":"1.295543701s","start":"2026-03-19T21:54:19.899737Z","end":"2026-03-19T21:54:21.195281Z","steps":["trace[1324946703] 'agreement among raft nodes before linearized reading' (duration: 1.294699047s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.19525Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:19.899677Z","time spent":"1.295556631s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":45,"response count":1,"response size":503,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" "} {"level":"warn","ts":"2026-03-19T21:54:21.195314Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:19.899724Z","time spent":"1.295584642s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":60,"response count":1,"response size":516,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" "} {"level":"warn","ts":"2026-03-19T21:54:21.987961Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000399721s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-19T21:54:21.988051Z","caller":"traceutil/trace.go:171","msg":"trace[1761051134] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000529205s","start":"2026-03-19T21:54:19.987503Z","end":"2026-03-19T21:54:21.988032Z","steps":["trace[1761051134] 'agreement among raft nodes before linearized reading' (duration: 2.000420832s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:21.988098Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:19.987489Z","time spent":"2.000599247s","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/19 21:54:21 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-19T21:54:22.25257Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.0000193s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-19T21:54:22.252661Z","caller":"traceutil/trace.go:171","msg":"trace[1657594379] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000155564s","start":"2026-03-19T21:54:20.252483Z","end":"2026-03-19T21:54:22.252639Z","steps":["trace[1657594379] 'agreement among raft nodes before linearized reading' (duration: 2.000052991s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:22.252709Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:20.252458Z","time spent":"2.000240857s","remote":"127.0.0.1:56892","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/19 21:54:22 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-19T21:54:22.355368Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.110406332s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:54:22.355449Z","caller":"traceutil/trace.go:171","msg":"trace[1372809123] linearizableReadLoop","detail":"{readStateIndex:1922; appliedIndex:1921; }","duration":"1.161410754s","start":"2026-03-19T21:54:21.194023Z","end":"2026-03-19T21:54:22.355434Z","steps":["trace[1372809123] 'read index received' (duration: 50.734054ms)","trace[1372809123] 'applied index is now lower than readState.Index' (duration: 1.11067552s)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:54:22.355512Z","caller":"traceutil/trace.go:171","msg":"trace[2109048015] transaction","detail":"{read_only:false; response_revision:1850; number_of_response:1; }","duration":"2.112490081s","start":"2026-03-19T21:54:20.243011Z","end":"2026-03-19T21:54:22.355501Z","steps":["trace[2109048015] 'process raft request' (duration: 1.001708637s)","trace[2109048015] 'compare' (duration: 1.110215587s)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:54:22.355819Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.400265662s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/deployments/openstack/percona-xtradb-pitr\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:54:22.355879Z","caller":"traceutil/trace.go:171","msg":"trace[1784573388] range","detail":"{range_begin:/registry/deployments/openstack/percona-xtradb-pitr; range_end:; response_count:0; response_revision:1850; }","duration":"2.400348654s","start":"2026-03-19T21:54:19.955516Z","end":"2026-03-19T21:54:22.355865Z","steps":["trace[1784573388] 'agreement among raft nodes before linearized reading' (duration: 2.400230831s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:22.355867Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.712023584s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumidentities/\" range_end:\"/registry/cilium.io/ciliumidentities0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"warn","ts":"2026-03-19T21:54:22.355919Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:19.9555Z","time spent":"2.400407625s","remote":"127.0.0.1:57282","response type":"/etcdserverpb.KV/Range","request count":0,"request size":53,"response count":0,"response size":29,"request content":"key:\"/registry/deployments/openstack/percona-xtradb-pitr\" "} {"level":"warn","ts":"2026-03-19T21:54:22.355916Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.282069798s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumendpoints/\" range_end:\"/registry/cilium.io/ciliumendpoints0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-03-19T21:54:22.355979Z","caller":"traceutil/trace.go:171","msg":"trace[1567452387] range","detail":"{range_begin:/registry/cilium.io/ciliumidentities/; range_end:/registry/cilium.io/ciliumidentities0; response_count:0; response_revision:1850; }","duration":"1.712159368s","start":"2026-03-19T21:54:20.643772Z","end":"2026-03-19T21:54:22.355931Z","steps":["trace[1567452387] 'agreement among raft nodes before linearized reading' (duration: 1.711966483s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:22.35603Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:20.643751Z","time spent":"1.712268252s","remote":"127.0.0.1:52142","response type":"/etcdserverpb.KV/Range","request count":0,"request size":80,"response count":15,"response size":31,"request content":"key:\"/registry/cilium.io/ciliumidentities/\" range_end:\"/registry/cilium.io/ciliumidentities0\" count_only:true "} {"level":"info","ts":"2026-03-19T21:54:22.356043Z","caller":"traceutil/trace.go:171","msg":"trace[2087812141] range","detail":"{range_begin:/registry/cilium.io/ciliumendpoints/; range_end:/registry/cilium.io/ciliumendpoints0; response_count:0; response_revision:1850; }","duration":"1.282255063s","start":"2026-03-19T21:54:21.073751Z","end":"2026-03-19T21:54:22.356006Z","steps":["trace[2087812141] 'agreement among raft nodes before linearized reading' (duration: 1.281991856s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:22.356102Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.073731Z","time spent":"1.282359117s","remote":"127.0.0.1:52100","response type":"/etcdserverpb.KV/Range","request count":0,"request size":78,"response count":12,"response size":31,"request content":"key:\"/registry/cilium.io/ciliumendpoints/\" range_end:\"/registry/cilium.io/ciliumendpoints0\" count_only:true "} {"level":"warn","ts":"2026-03-19T21:54:22.356442Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:20.24297Z","time spent":"2.112564294s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":672,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:22.856388Z","caller":"etcdserver/v3_server.go:897","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8713512037097300235,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-19T21:54:23.066194Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"1.82146705s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-19T21:54:24.21603Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"1.090596879s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-19T21:54:24.2595Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000184326s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-19T21:54:24.259595Z","caller":"traceutil/trace.go:171","msg":"trace[174168218] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.00030437s","start":"2026-03-19T21:54:22.259254Z","end":"2026-03-19T21:54:24.259558Z","steps":["trace[174168218] 'agreement among raft nodes before linearized reading' (duration: 2.000200757s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:24.259646Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:22.259247Z","time spent":"2.000389452s","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-19T21:54:25.386402Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.319963698s","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:41"} {"level":"warn","ts":"2026-03-19T21:54:25.38659Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.196398Z","time spent":"4.190185715s","remote":"127.0.0.1:56898","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-19T21:54:25.386871Z","caller":"traceutil/trace.go:171","msg":"trace[560575641] transaction","detail":"{read_only:false; response_revision:1851; number_of_response:1; }","duration":"4.188099325s","start":"2026-03-19T21:54:21.198737Z","end":"2026-03-19T21:54:25.386837Z","steps":["trace[560575641] 'process raft request' (duration: 4.187769436s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.387052Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.198702Z","time spent":"4.188227629s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":418,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:25.387177Z","caller":"traceutil/trace.go:171","msg":"trace[1819763416] transaction","detail":"{read_only:false; response_revision:1852; number_of_response:1; }","duration":"4.187113197s","start":"2026-03-19T21:54:21.200012Z","end":"2026-03-19T21:54:25.387125Z","steps":["trace[1819763416] 'process raft request' (duration: 4.186701876s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.387415Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.2Z","time spent":"4.187278812s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":422,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:25.387679Z","caller":"traceutil/trace.go:171","msg":"trace[424993155] transaction","detail":"{read_only:false; response_revision:1853; number_of_response:1; }","duration":"4.187506028s","start":"2026-03-19T21:54:21.200149Z","end":"2026-03-19T21:54:25.387655Z","steps":["trace[424993155] 'process raft request' (duration: 4.186903751s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:25.387694Z","caller":"traceutil/trace.go:171","msg":"trace[2100616354] transaction","detail":"{read_only:false; response_revision:1854; number_of_response:1; }","duration":"4.186745557s","start":"2026-03-19T21:54:21.20093Z","end":"2026-03-19T21:54:25.387676Z","steps":["trace[2100616354] 'process raft request' (duration: 4.1864908s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:25.387778Z","caller":"traceutil/trace.go:171","msg":"trace[325264932] transaction","detail":"{read_only:false; response_revision:1855; number_of_response:1; }","duration":"4.186744836s","start":"2026-03-19T21:54:21.201004Z","end":"2026-03-19T21:54:25.387749Z","steps":["trace[325264932] 'process raft request' (duration: 4.186623922s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.387926Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.200999Z","time spent":"4.186847299s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":540,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:25.387956Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.200123Z","time spent":"4.187614541s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":491,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:25.388007Z","caller":"traceutil/trace.go:171","msg":"trace[699475035] transaction","detail":"{read_only:false; response_revision:1856; number_of_response:1; }","duration":"4.185249654s","start":"2026-03-19T21:54:21.202747Z","end":"2026-03-19T21:54:25.387997Z","steps":["trace[699475035] 'process raft request' (duration: 4.184926115s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.387803Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.200914Z","time spent":"4.186825109s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":464,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:25.388074Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.202738Z","time spent":"4.185304575s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":477,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:25.388173Z","caller":"traceutil/trace.go:171","msg":"trace[245723976] transaction","detail":"{read_only:false; response_revision:1857; number_of_response:1; }","duration":"4.105630327s","start":"2026-03-19T21:54:21.282526Z","end":"2026-03-19T21:54:25.388156Z","steps":["trace[245723976] 'process raft request' (duration: 4.105184665s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.38829Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.282499Z","time spent":"4.105745901s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":475,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:25.388761Z","caller":"traceutil/trace.go:171","msg":"trace[1670298818] linearizableReadLoop","detail":"{readStateIndex:1932; appliedIndex:1922; }","duration":"3.033263407s","start":"2026-03-19T21:54:22.355471Z","end":"2026-03-19T21:54:25.388734Z","steps":["trace[1670298818] 'read index received' (duration: 710.955553ms)","trace[1670298818] 'applied index is now lower than readState.Index' (duration: 2.322306384s)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:54:25.389008Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.043928851s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cilium.io/ciliumexternalworkloads/\" range_end:\"/registry/cilium.io/ciliumexternalworkloads0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:54:25.389063Z","caller":"traceutil/trace.go:171","msg":"trace[1654908049] range","detail":"{range_begin:/registry/cilium.io/ciliumexternalworkloads/; range_end:/registry/cilium.io/ciliumexternalworkloads0; response_count:0; response_revision:1859; }","duration":"4.043988213s","start":"2026-03-19T21:54:21.345062Z","end":"2026-03-19T21:54:25.389051Z","steps":["trace[1654908049] 'agreement among raft nodes before linearized reading' (duration: 4.043892351s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.389096Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.344998Z","time spent":"4.044088896s","remote":"127.0.0.1:52162","response type":"/etcdserverpb.KV/Range","request count":0,"request size":94,"response count":0,"response size":29,"request content":"key:\"/registry/cilium.io/ciliumexternalworkloads/\" range_end:\"/registry/cilium.io/ciliumexternalworkloads0\" count_only:true "} {"level":"warn","ts":"2026-03-19T21:54:25.389097Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.185233414s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" ","response":"range_response_count:1 size:5444"} {"level":"info","ts":"2026-03-19T21:54:25.389155Z","caller":"traceutil/trace.go:171","msg":"trace[1296183271] range","detail":"{range_begin:/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb; range_end:; response_count:1; response_revision:1859; }","duration":"3.185341948s","start":"2026-03-19T21:54:22.203793Z","end":"2026-03-19T21:54:25.389135Z","steps":["trace[1296183271] 'agreement among raft nodes before linearized reading' (duration: 3.185147281s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.389213Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:22.203762Z","time spent":"3.1854363s","remote":"127.0.0.1:51270","response type":"/etcdserverpb.KV/Range","request count":0,"request size":74,"response count":1,"response size":5468,"request content":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" "} {"level":"warn","ts":"2026-03-19T21:54:25.389278Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.145459981s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/cert-manager/cert-manager-controller\" ","response":"range_response_count:1 size:528"} {"level":"info","ts":"2026-03-19T21:54:25.388786Z","caller":"traceutil/trace.go:171","msg":"trace[1960873285] transaction","detail":"{read_only:false; response_revision:1859; number_of_response:1; }","duration":"3.397642757s","start":"2026-03-19T21:54:21.991129Z","end":"2026-03-19T21:54:25.388772Z","steps":["trace[1960873285] 'process raft request' (duration: 3.397488303s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:25.389377Z","caller":"traceutil/trace.go:171","msg":"trace[1714672893] range","detail":"{range_begin:/registry/leases/cert-manager/cert-manager-controller; range_end:; response_count:1; response_revision:1859; }","duration":"3.145593785s","start":"2026-03-19T21:54:22.243732Z","end":"2026-03-19T21:54:25.389326Z","steps":["trace[1714672893] 'agreement among raft nodes before linearized reading' (duration: 3.145315176s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.389373Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.536201612s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets/\" range_end:\"/registry/secrets0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"warn","ts":"2026-03-19T21:54:25.389478Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:22.243712Z","time spent":"3.14575029s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":55,"response count":1,"response size":552,"request content":"key:\"/registry/leases/cert-manager/cert-manager-controller\" "} {"level":"warn","ts":"2026-03-19T21:54:25.389492Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.991104Z","time spent":"3.398273856s","remote":"127.0.0.1:56936","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":758,"response count":0,"response size":40,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-19T21:54:25.389515Z","caller":"traceutil/trace.go:171","msg":"trace[1355659284] range","detail":"{range_begin:/registry/secrets/; range_end:/registry/secrets0; response_count:0; response_revision:1859; }","duration":"3.536427029s","start":"2026-03-19T21:54:21.853071Z","end":"2026-03-19T21:54:25.389498Z","steps":["trace[1355659284] 'agreement among raft nodes before linearized reading' (duration: 3.536045698s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:25.388892Z","caller":"traceutil/trace.go:171","msg":"trace[854270023] transaction","detail":"{read_only:false; response_revision:1858; number_of_response:1; }","duration":"3.427142757s","start":"2026-03-19T21:54:21.961728Z","end":"2026-03-19T21:54:25.38887Z","steps":["trace[854270023] 'process raft request' (duration: 3.426211601s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:25.389741Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.961703Z","time spent":"3.427938901s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":509,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:25.38998Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:21.85305Z","time spent":"3.536564032s","remote":"127.0.0.1:56956","response type":"/etcdserverpb.KV/Range","request count":0,"request size":42,"response count":15,"response size":31,"request content":"key:\"/registry/secrets/\" range_end:\"/registry/secrets0\" count_only:true "} {"level":"warn","ts":"2026-03-19T21:54:25.571798Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"1.330161527s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-19T21:54:26.260866Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000281258s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-19T21:54:26.260935Z","caller":"traceutil/trace.go:171","msg":"trace[1769587034] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000396622s","start":"2026-03-19T21:54:24.260523Z","end":"2026-03-19T21:54:26.26092Z","steps":["trace[1769587034] 'agreement among raft nodes before linearized reading' (duration: 2.000307359s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.260971Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:24.260502Z","time spent":"2.000462024s","remote":"127.0.0.1:56892","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/19 21:54:26 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-19T21:54:26.582646Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000100404s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-19T21:54:26.582746Z","caller":"traceutil/trace.go:171","msg":"trace[1384012829] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000231017s","start":"2026-03-19T21:54:24.582497Z","end":"2026-03-19T21:54:26.582728Z","steps":["trace[1384012829] 'agreement among raft nodes before linearized reading' (duration: 2.000097264s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.582794Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:24.582447Z","time spent":"2.00033874s","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/19 21:54:26 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-19T21:54:26.652274Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.079983656s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:54:26.652589Z","caller":"traceutil/trace.go:171","msg":"trace[1123728131] transaction","detail":"{read_only:false; response_revision:1862; number_of_response:1; }","duration":"2.730744578s","start":"2026-03-19T21:54:23.921827Z","end":"2026-03-19T21:54:26.652572Z","steps":["trace[1123728131] 'process raft request' (duration: 2.730651645s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.652675Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:23.921805Z","time spent":"2.73082758s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":521,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:54:26.652887Z","caller":"traceutil/trace.go:171","msg":"trace[1961168297] transaction","detail":"{read_only:false; response_revision:1860; number_of_response:1; }","duration":"3.451580873s","start":"2026-03-19T21:54:23.201257Z","end":"2026-03-19T21:54:26.652838Z","steps":["trace[1961168297] 'process raft request' (duration: 2.370914927s)","trace[1961168297] 'compare' (duration: 1.079644086s)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:54:26.652866Z","caller":"traceutil/trace.go:171","msg":"trace[1639413661] transaction","detail":"{read_only:false; response_revision:1861; number_of_response:1; }","duration":"3.029380128s","start":"2026-03-19T21:54:23.623307Z","end":"2026-03-19T21:54:26.652687Z","steps":["trace[1639413661] 'process raft request' (duration: 3.029068538s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.653112Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:23.201222Z","time spent":"3.451775468s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":554,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:26.653191Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:23.623279Z","time spent":"3.029729938s","remote":"127.0.0.1:57026","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":8393,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:26.779634Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"127.078616ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:29"} {"level":"info","ts":"2026-03-19T21:54:26.780093Z","caller":"traceutil/trace.go:171","msg":"trace[826419782] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1863; }","duration":"1.392556022s","start":"2026-03-19T21:54:25.3875Z","end":"2026-03-19T21:54:26.780056Z","steps":["trace[826419782] 'process raft request' (duration: 1.392231933s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:26.780155Z","caller":"traceutil/trace.go:171","msg":"trace[1258940756] linearizableReadLoop","detail":"{readStateIndex:1937; appliedIndex:1932; }","duration":"1.391099961s","start":"2026-03-19T21:54:25.389032Z","end":"2026-03-19T21:54:26.780132Z","steps":["trace[1258940756] 'read index received' (duration: 183.083751ms)","trace[1258940756] 'applied index is now lower than readState.Index' (duration: 1.20801482s)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:54:26.780313Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:25.387481Z","time spent":"1.392701737s","remote":"127.0.0.1:56898","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":40,"response count":0,"response size":38,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:26.780403Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.894389205s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/\" range_end:\"/registry/leases0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-03-19T21:54:26.780512Z","caller":"traceutil/trace.go:171","msg":"trace[1864439106] range","detail":"{range_begin:/registry/leases/; range_end:/registry/leases0; response_count:0; response_revision:1864; }","duration":"3.894593281s","start":"2026-03-19T21:54:22.885896Z","end":"2026-03-19T21:54:26.78049Z","steps":["trace[1864439106] 'agreement among raft nodes before linearized reading' (duration: 3.894301673s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.780375Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.421205229s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs/openstack/percona-xtradb-pitr\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:54:26.780614Z","caller":"traceutil/trace.go:171","msg":"trace[285269213] transaction","detail":"{read_only:false; response_revision:1864; number_of_response:1; }","duration":"1.386423048s","start":"2026-03-19T21:54:25.394164Z","end":"2026-03-19T21:54:26.780587Z","steps":["trace[285269213] 'process raft request' (duration: 1.38577664s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.780622Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.387874068s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/openstack/percona-xtradb-haproxy-0.189e5cb680114d93\" ","response":"range_response_count:1 size:846"} {"level":"warn","ts":"2026-03-19T21:54:26.780656Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"500.095653ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:54:26.780691Z","caller":"traceutil/trace.go:171","msg":"trace[1002475999] range","detail":"{range_begin:/registry/events/openstack/percona-xtradb-haproxy-0.189e5cb680114d93; range_end:; response_count:1; response_revision:1864; }","duration":"1.387982421s","start":"2026-03-19T21:54:25.392691Z","end":"2026-03-19T21:54:26.780674Z","steps":["trace[1002475999] 'agreement among raft nodes before linearized reading' (duration: 1.387752985s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.780713Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"512.070364ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-19T21:54:26.780743Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:25.39267Z","time spent":"1.388062154s","remote":"127.0.0.1:56936","response type":"/etcdserverpb.KV/Range","request count":0,"request size":70,"response count":1,"response size":870,"request content":"key:\"/registry/events/openstack/percona-xtradb-haproxy-0.189e5cb680114d93\" "} {"level":"warn","ts":"2026-03-19T21:54:26.780767Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:25.394147Z","time spent":"1.386532371s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":513,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:54:26.780758Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"384.106062ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" ","response":"range_response_count:1 size:5444"} {"level":"info","ts":"2026-03-19T21:54:26.780733Z","caller":"traceutil/trace.go:171","msg":"trace[548557253] range","detail":"{range_begin:/registry/horizontalpodautoscalers/; range_end:/registry/horizontalpodautoscalers0; response_count:0; response_revision:1864; }","duration":"500.175104ms","start":"2026-03-19T21:54:26.280517Z","end":"2026-03-19T21:54:26.780692Z","steps":["trace[548557253] 'agreement among raft nodes before linearized reading' (duration: 500.103412ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:26.780817Z","caller":"traceutil/trace.go:171","msg":"trace[693572403] range","detail":"{range_begin:/registry/services/specs/openstack/percona-xtradb-pitr; range_end:; response_count:0; response_revision:1864; }","duration":"4.421659292s","start":"2026-03-19T21:54:22.359089Z","end":"2026-03-19T21:54:26.780749Z","steps":["trace[693572403] 'agreement among raft nodes before linearized reading' (duration: 4.421221479s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:26.780841Z","caller":"traceutil/trace.go:171","msg":"trace[224324667] range","detail":"{range_begin:/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb; range_end:; response_count:1; response_revision:1864; }","duration":"384.236786ms","start":"2026-03-19T21:54:26.396586Z","end":"2026-03-19T21:54:26.780823Z","steps":["trace[224324667] 'agreement among raft nodes before linearized reading' (duration: 383.991689ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.780856Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:26.280499Z","time spent":"500.339179ms","remote":"127.0.0.1:57056","response type":"/etcdserverpb.KV/Range","request count":0,"request size":76,"response count":0,"response size":29,"request content":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true "} {"level":"warn","ts":"2026-03-19T21:54:26.780888Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:26.396564Z","time spent":"384.310397ms","remote":"127.0.0.1:51270","response type":"/etcdserverpb.KV/Range","request count":0,"request size":74,"response count":1,"response size":5468,"request content":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/openstack/percona-xtradb\" "} {"level":"warn","ts":"2026-03-19T21:54:26.780494Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.402539706s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rabbitmq.com/queues/\" range_end:\"/registry/rabbitmq.com/queues0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-19T21:54:26.780933Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:22.359069Z","time spent":"4.421820216s","remote":"127.0.0.1:57040","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":29,"request content":"key:\"/registry/services/specs/openstack/percona-xtradb-pitr\" "} {"level":"warn","ts":"2026-03-19T21:54:26.780591Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:22.885875Z","time spent":"3.894682714s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":13,"response size":31,"request content":"key:\"/registry/leases/\" range_end:\"/registry/leases0\" count_only:true "} {"level":"info","ts":"2026-03-19T21:54:26.780968Z","caller":"traceutil/trace.go:171","msg":"trace[1565548527] range","detail":"{range_begin:/registry/rabbitmq.com/queues/; range_end:/registry/rabbitmq.com/queues0; response_count:0; response_revision:1864; }","duration":"2.403115602s","start":"2026-03-19T21:54:24.377838Z","end":"2026-03-19T21:54:26.780954Z","steps":["trace[1565548527] 'agreement among raft nodes before linearized reading' (duration: 2.40232803s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.781013Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:24.377814Z","time spent":"2.403183935s","remote":"127.0.0.1:51106","response type":"/etcdserverpb.KV/Range","request count":0,"request size":66,"response count":0,"response size":29,"request content":"key:\"/registry/rabbitmq.com/queues/\" range_end:\"/registry/rabbitmq.com/queues0\" count_only:true "} {"level":"info","ts":"2026-03-19T21:54:26.780789Z","caller":"traceutil/trace.go:171","msg":"trace[539750472] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1864; }","duration":"512.188497ms","start":"2026-03-19T21:54:26.268586Z","end":"2026-03-19T21:54:26.780775Z","steps":["trace[539750472] 'agreement among raft nodes before linearized reading' (duration: 512.045333ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:54:26.781125Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:54:26.268574Z","time spent":"512.538047ms","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":29,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-19T21:54:27.02391Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"126.562432ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:54:27.024024Z","caller":"traceutil/trace.go:171","msg":"trace[260405886] linearizableReadLoop","detail":"{readStateIndex:1942; appliedIndex:1941; }","duration":"196.800271ms","start":"2026-03-19T21:54:26.827206Z","end":"2026-03-19T21:54:27.024006Z","steps":["trace[260405886] 'read index received' (duration: 69.978732ms)","trace[260405886] 'applied index is now lower than readState.Index' (duration: 126.819229ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:54:27.024193Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"196.995917ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" ","response":"range_response_count:1 size:421"} {"level":"info","ts":"2026-03-19T21:54:27.024247Z","caller":"traceutil/trace.go:171","msg":"trace[2040794397] transaction","detail":"{read_only:false; response_revision:1867; number_of_response:1; }","duration":"208.73036ms","start":"2026-03-19T21:54:26.815465Z","end":"2026-03-19T21:54:27.024195Z","steps":["trace[2040794397] 'process raft request' (duration: 81.804568ms)","trace[2040794397] 'compare' (duration: 126.236793ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:54:27.024275Z","caller":"traceutil/trace.go:171","msg":"trace[1816586716] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1867; }","duration":"197.103599ms","start":"2026-03-19T21:54:26.827119Z","end":"2026-03-19T21:54:27.024223Z","steps":["trace[1816586716] 'agreement among raft nodes before linearized reading' (duration: 196.932184ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:27.552634Z","caller":"traceutil/trace.go:171","msg":"trace[1677235535] transaction","detail":"{read_only:false; response_revision:1871; number_of_response:1; }","duration":"157.444501ms","start":"2026-03-19T21:54:27.395154Z","end":"2026-03-19T21:54:27.552599Z","steps":["trace[1677235535] 'process raft request' (duration: 123.298399ms)","trace[1677235535] 'compare' (duration: 34.025489ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:54:27.608822Z","caller":"traceutil/trace.go:171","msg":"trace[969308060] transaction","detail":"{read_only:false; response_revision:1872; number_of_response:1; }","duration":"211.87679ms","start":"2026-03-19T21:54:27.396921Z","end":"2026-03-19T21:54:27.608798Z","steps":["trace[969308060] 'process raft request' (duration: 211.714366ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:27.609186Z","caller":"traceutil/trace.go:171","msg":"trace[1228402640] transaction","detail":"{read_only:false; response_revision:1873; number_of_response:1; }","duration":"212.053775ms","start":"2026-03-19T21:54:27.397095Z","end":"2026-03-19T21:54:27.609148Z","steps":["trace[1228402640] 'process raft request' (duration: 211.668514ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:27.609195Z","caller":"traceutil/trace.go:171","msg":"trace[173850003] transaction","detail":"{read_only:false; response_revision:1874; number_of_response:1; }","duration":"212.007104ms","start":"2026-03-19T21:54:27.397167Z","end":"2026-03-19T21:54:27.609174Z","steps":["trace[173850003] 'process raft request' (duration: 211.842509ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:27.609233Z","caller":"traceutil/trace.go:171","msg":"trace[671667597] transaction","detail":"{read_only:false; response_revision:1875; number_of_response:1; }","duration":"211.135018ms","start":"2026-03-19T21:54:27.398087Z","end":"2026-03-19T21:54:27.609222Z","steps":["trace[671667597] 'process raft request' (duration: 211.022825ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:27.609337Z","caller":"traceutil/trace.go:171","msg":"trace[1159304615] transaction","detail":"{read_only:false; response_revision:1876; number_of_response:1; }","duration":"210.275884ms","start":"2026-03-19T21:54:27.399051Z","end":"2026-03-19T21:54:27.609327Z","steps":["trace[1159304615] 'process raft request' (duration: 210.10998ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:27.609503Z","caller":"traceutil/trace.go:171","msg":"trace[523473850] transaction","detail":"{read_only:false; response_revision:1877; number_of_response:1; }","duration":"210.166212ms","start":"2026-03-19T21:54:27.399241Z","end":"2026-03-19T21:54:27.609407Z","steps":["trace[523473850] 'process raft request' (duration: 209.969706ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:29.806521Z","caller":"traceutil/trace.go:171","msg":"trace[377493796] transaction","detail":"{read_only:false; response_revision:1883; number_of_response:1; }","duration":"187.63328ms","start":"2026-03-19T21:54:29.618846Z","end":"2026-03-19T21:54:29.806479Z","steps":["trace[377493796] 'process raft request' (duration: 187.518877ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:29.806684Z","caller":"traceutil/trace.go:171","msg":"trace[730070882] transaction","detail":"{read_only:false; response_revision:1882; number_of_response:1; }","duration":"187.843935ms","start":"2026-03-19T21:54:29.618794Z","end":"2026-03-19T21:54:29.806638Z","steps":["trace[730070882] 'process raft request' (duration: 165.626022ms)","trace[730070882] 'compare' (duration: 21.804452ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:54:29.806726Z","caller":"traceutil/trace.go:171","msg":"trace[1693115291] transaction","detail":"{read_only:false; response_revision:1886; number_of_response:1; }","duration":"187.746872ms","start":"2026-03-19T21:54:29.61896Z","end":"2026-03-19T21:54:29.806707Z","steps":["trace[1693115291] 'process raft request' (duration: 187.67148ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:29.806792Z","caller":"traceutil/trace.go:171","msg":"trace[1729719541] transaction","detail":"{read_only:false; response_revision:1885; number_of_response:1; }","duration":"187.809015ms","start":"2026-03-19T21:54:29.618922Z","end":"2026-03-19T21:54:29.806731Z","steps":["trace[1729719541] 'process raft request' (duration: 187.62491ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:54:29.806827Z","caller":"traceutil/trace.go:171","msg":"trace[1196799731] transaction","detail":"{read_only:false; response_revision:1884; number_of_response:1; }","duration":"187.913038ms","start":"2026-03-19T21:54:29.618885Z","end":"2026-03-19T21:54:29.806798Z","steps":["trace[1196799731] 'process raft request' (duration: 187.551718ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:55:02.214943Z","caller":"traceutil/trace.go:171","msg":"trace[977078922] transaction","detail":"{read_only:false; response_revision:2034; number_of_response:1; }","duration":"105.461482ms","start":"2026-03-19T21:55:02.109452Z","end":"2026-03-19T21:55:02.214913Z","steps":["trace[977078922] 'process raft request' (duration: 68.409738ms)","trace[977078922] 'compare' (duration: 36.839708ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:55:02.21545Z","caller":"traceutil/trace.go:171","msg":"trace[1854272972] transaction","detail":"{read_only:false; response_revision:2035; number_of_response:1; }","duration":"103.868776ms","start":"2026-03-19T21:55:02.11155Z","end":"2026-03-19T21:55:02.215419Z","steps":["trace[1854272972] 'process raft request' (duration: 103.29961ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:55:02.21569Z","caller":"traceutil/trace.go:171","msg":"trace[593734565] transaction","detail":"{read_only:false; response_revision:2036; number_of_response:1; }","duration":"103.97812ms","start":"2026-03-19T21:55:02.111669Z","end":"2026-03-19T21:55:02.215647Z","steps":["trace[593734565] 'process raft request' (duration: 103.504576ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:55:02.216131Z","caller":"traceutil/trace.go:171","msg":"trace[1522322424] transaction","detail":"{read_only:false; response_revision:2037; number_of_response:1; }","duration":"104.00579ms","start":"2026-03-19T21:55:02.112105Z","end":"2026-03-19T21:55:02.21611Z","steps":["trace[1522322424] 'process raft request' (duration: 103.430784ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:55:02.615205Z","caller":"traceutil/trace.go:171","msg":"trace[2143916928] transaction","detail":"{read_only:false; response_revision:2043; number_of_response:1; }","duration":"104.173525ms","start":"2026-03-19T21:55:02.511001Z","end":"2026-03-19T21:55:02.615175Z","steps":["trace[2143916928] 'process raft request' (duration: 50.410695ms)","trace[2143916928] 'compare' (duration: 53.609745ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:55:08.375995Z","caller":"traceutil/trace.go:171","msg":"trace[1391218491] transaction","detail":"{read_only:false; response_revision:2073; number_of_response:1; }","duration":"107.556041ms","start":"2026-03-19T21:55:08.268404Z","end":"2026-03-19T21:55:08.37596Z","steps":["trace[1391218491] 'process raft request' (duration: 28.230744ms)","trace[1391218491] 'compare' (duration: 79.132232ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:55:42.455737Z","caller":"traceutil/trace.go:171","msg":"trace[1413528255] transaction","detail":"{read_only:false; response_revision:2400; number_of_response:1; }","duration":"126.528515ms","start":"2026-03-19T21:55:42.329173Z","end":"2026-03-19T21:55:42.455702Z","steps":["trace[1413528255] 'process raft request' (duration: 126.286258ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:10.365335Z","caller":"traceutil/trace.go:171","msg":"trace[1229224246] linearizableReadLoop","detail":"{readStateIndex:2664; appliedIndex:2663; }","duration":"111.846489ms","start":"2026-03-19T21:56:10.253462Z","end":"2026-03-19T21:56:10.365309Z","steps":["trace[1229224246] 'read index received' (duration: 111.678165ms)","trace[1229224246] 'applied index is now lower than readState.Index' (duration: 167.124µs)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:56:10.365396Z","caller":"traceutil/trace.go:171","msg":"trace[572433885] transaction","detail":"{read_only:false; response_revision:2559; number_of_response:1; }","duration":"129.436926ms","start":"2026-03-19T21:56:10.235926Z","end":"2026-03-19T21:56:10.365363Z","steps":["trace[572433885] 'process raft request' (duration: 129.21475ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:10.365525Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"112.011795ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:56:10.365563Z","caller":"traceutil/trace.go:171","msg":"trace[297009102] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:2559; }","duration":"112.191751ms","start":"2026-03-19T21:56:10.25336Z","end":"2026-03-19T21:56:10.365552Z","steps":["trace[297009102] 'agreement among raft nodes before linearized reading' (duration: 112.070547ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:11.157344Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"105.369867ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:56:11.157467Z","caller":"traceutil/trace.go:171","msg":"trace[1267206584] linearizableReadLoop","detail":"{readStateIndex:2666; appliedIndex:2665; }","duration":"501.123047ms","start":"2026-03-19T21:56:10.656326Z","end":"2026-03-19T21:56:11.157449Z","steps":["trace[1267206584] 'read index received' (duration: 395.45887ms)","trace[1267206584] 'applied index is now lower than readState.Index' (duration: 105.662437ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:56:11.157566Z","caller":"traceutil/trace.go:171","msg":"trace[564764423] transaction","detail":"{read_only:false; response_revision:2561; number_of_response:1; }","duration":"537.896563ms","start":"2026-03-19T21:56:10.619656Z","end":"2026-03-19T21:56:11.157553Z","steps":["trace[564764423] 'process raft request' (duration: 432.219717ms)","trace[564764423] 'compare' (duration: 105.08003ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:11.157634Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:10.619637Z","time spent":"537.964335ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":475,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:56:11.158211Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"501.887458ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:433"} {"level":"warn","ts":"2026-03-19T21:56:11.158203Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"255.462116ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" ","response":"range_response_count:1 size:437"} {"level":"warn","ts":"2026-03-19T21:56:11.158207Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"282.058675ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" ","response":"range_response_count:1 size:479"} {"level":"warn","ts":"2026-03-19T21:56:11.158294Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"440.978413ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" ","response":"range_response_count:1 size:492"} {"level":"warn","ts":"2026-03-19T21:56:11.158295Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"367.023509ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" ","response":"range_response_count:1 size:555"} {"level":"info","ts":"2026-03-19T21:56:11.158334Z","caller":"traceutil/trace.go:171","msg":"trace[567172633] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:2561; }","duration":"502.052364ms","start":"2026-03-19T21:56:10.656261Z","end":"2026-03-19T21:56:11.158314Z","steps":["trace[567172633] 'agreement among raft nodes before linearized reading' (duration: 501.746865ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:11.15835Z","caller":"traceutil/trace.go:171","msg":"trace[1035159834] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:2561; }","duration":"255.651913ms","start":"2026-03-19T21:56:10.902674Z","end":"2026-03-19T21:56:11.158326Z","steps":["trace[1035159834] 'agreement among raft nodes before linearized reading' (duration: 255.372614ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:11.158373Z","caller":"traceutil/trace.go:171","msg":"trace[1369531004] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:2561; }","duration":"441.162858ms","start":"2026-03-19T21:56:10.717192Z","end":"2026-03-19T21:56:11.158354Z","steps":["trace[1369531004] 'agreement among raft nodes before linearized reading' (duration: 440.859409ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:11.158377Z","caller":"traceutil/trace.go:171","msg":"trace[1946473305] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:2561; }","duration":"282.255782ms","start":"2026-03-19T21:56:10.876084Z","end":"2026-03-19T21:56:11.15834Z","steps":["trace[1946473305] 'agreement among raft nodes before linearized reading' (duration: 281.971213ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:11.158386Z","caller":"traceutil/trace.go:171","msg":"trace[1631206392] range","detail":"{range_begin:/registry/leases/openstack/rabbitmq-cluster-operator-leader-election; range_end:; response_count:1; response_revision:2561; }","duration":"367.125283ms","start":"2026-03-19T21:56:10.791239Z","end":"2026-03-19T21:56:11.158365Z","steps":["trace[1631206392] 'agreement among raft nodes before linearized reading' (duration: 366.829154ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:11.158394Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:10.65624Z","time spent":"502.137846ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":44,"response count":1,"response size":457,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" "} {"level":"warn","ts":"2026-03-19T21:56:11.15842Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:10.717171Z","time spent":"441.24093ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":60,"response count":1,"response size":516,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" "} {"level":"warn","ts":"2026-03-19T21:56:11.158426Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:10.791171Z","time spent":"367.247246ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":70,"response count":1,"response size":579,"request content":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" "} {"level":"warn","ts":"2026-03-19T21:56:11.634906Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"273.122695ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:56:11.635205Z","caller":"traceutil/trace.go:171","msg":"trace[817008683] transaction","detail":"{read_only:false; response_revision:2565; number_of_response:1; }","duration":"471.763321ms","start":"2026-03-19T21:56:11.163419Z","end":"2026-03-19T21:56:11.635183Z","steps":["trace[817008683] 'process raft request' (duration: 471.691939ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:11.635215Z","caller":"traceutil/trace.go:171","msg":"trace[1992232882] transaction","detail":"{read_only:false; response_revision:2563; number_of_response:1; }","duration":"472.207643ms","start":"2026-03-19T21:56:11.16295Z","end":"2026-03-19T21:56:11.635157Z","steps":["trace[1992232882] 'process raft request' (duration: 198.722998ms)","trace[1992232882] 'compare' (duration: 272.680692ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:11.635286Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:11.163415Z","time spent":"471.834622ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":464,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:56:11.635297Z","caller":"traceutil/trace.go:171","msg":"trace[337973730] transaction","detail":"{read_only:false; response_revision:2566; number_of_response:1; }","duration":"471.603376ms","start":"2026-03-19T21:56:11.163678Z","end":"2026-03-19T21:56:11.635281Z","steps":["trace[337973730] 'process raft request' (duration: 471.467313ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:11.635444Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:11.163671Z","time spent":"471.679718ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":418,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:56:11.635459Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:11.162917Z","time spent":"472.412099ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":477,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:56:11.63523Z","caller":"traceutil/trace.go:171","msg":"trace[214293446] transaction","detail":"{read_only:false; response_revision:2564; number_of_response:1; }","duration":"471.830363ms","start":"2026-03-19T21:56:11.163356Z","end":"2026-03-19T21:56:11.635186Z","steps":["trace[214293446] 'process raft request' (duration: 471.670638ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:11.635598Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:11.163337Z","time spent":"472.234043ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":422,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:56:11.677429Z","caller":"traceutil/trace.go:171","msg":"trace[185746074] transaction","detail":"{read_only:false; response_revision:2567; number_of_response:1; }","duration":"465.835273ms","start":"2026-03-19T21:56:11.211566Z","end":"2026-03-19T21:56:11.677402Z","steps":["trace[185746074] 'process raft request' (duration: 465.697249ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:11.677718Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:11.211546Z","time spent":"465.964536ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":554,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:56:11.959354Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"104.069241ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/cert-manager/cert-manager-controller\" ","response":"range_response_count:1 size:529"} {"level":"info","ts":"2026-03-19T21:56:11.959477Z","caller":"traceutil/trace.go:171","msg":"trace[1985483125] range","detail":"{range_begin:/registry/leases/cert-manager/cert-manager-controller; range_end:; response_count:1; response_revision:2567; }","duration":"104.242636ms","start":"2026-03-19T21:56:11.855196Z","end":"2026-03-19T21:56:11.959438Z","steps":["trace[1985483125] 'range keys from in-memory index tree' (duration: 103.805804ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:12.076027Z","caller":"traceutil/trace.go:171","msg":"trace[1584468879] transaction","detail":"{read_only:false; response_revision:2568; number_of_response:1; }","duration":"112.249522ms","start":"2026-03-19T21:56:11.96375Z","end":"2026-03-19T21:56:12.076Z","steps":["trace[1584468879] 'process raft request' (duration: 112.078617ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:12.35709Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"103.66227ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:56:12.357261Z","caller":"traceutil/trace.go:171","msg":"trace[207625027] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:2568; }","duration":"103.908257ms","start":"2026-03-19T21:56:12.253333Z","end":"2026-03-19T21:56:12.357241Z","steps":["trace[207625027] 'range keys from in-memory index tree' (duration: 103.565087ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:12.829061Z","caller":"traceutil/trace.go:171","msg":"trace[429123608] linearizableReadLoop","detail":"{readStateIndex:2675; appliedIndex:2674; }","duration":"286.396798ms","start":"2026-03-19T21:56:12.542639Z","end":"2026-03-19T21:56:12.829036Z","steps":["trace[429123608] 'read index received' (duration: 208.280587ms)","trace[429123608] 'applied index is now lower than readState.Index' (duration: 78.114731ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:56:12.829217Z","caller":"traceutil/trace.go:171","msg":"trace[1151591448] transaction","detail":"{read_only:false; response_revision:2570; number_of_response:1; }","duration":"336.420027ms","start":"2026-03-19T21:56:12.492769Z","end":"2026-03-19T21:56:12.829189Z","steps":["trace[1151591448] 'process raft request' (duration: 258.231884ms)","trace[1151591448] 'compare' (duration: 77.73503ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:12.829458Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"286.798839ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs/auth-system/keycloak-headless\" ","response":"range_response_count:1 size:1369"} {"level":"info","ts":"2026-03-19T21:56:12.82954Z","caller":"traceutil/trace.go:171","msg":"trace[1349440243] range","detail":"{range_begin:/registry/services/specs/auth-system/keycloak-headless; range_end:; response_count:1; response_revision:2570; }","duration":"286.950333ms","start":"2026-03-19T21:56:12.542572Z","end":"2026-03-19T21:56:12.829523Z","steps":["trace[1349440243] 'agreement among raft nodes before linearized reading' (duration: 286.682686ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:12.82967Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:12.492752Z","time spent":"336.610983ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":543,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:56:12.829724Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"204.334786ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" ","response":"range_response_count:1 size:506"} {"level":"info","ts":"2026-03-19T21:56:12.829816Z","caller":"traceutil/trace.go:171","msg":"trace[584928661] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:2570; }","duration":"204.47955ms","start":"2026-03-19T21:56:12.625316Z","end":"2026-03-19T21:56:12.829795Z","steps":["trace[584928661] 'agreement among raft nodes before linearized reading' (duration: 204.182521ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:13.696757Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"720.779816ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:56:13.696872Z","caller":"traceutil/trace.go:171","msg":"trace[144247570] linearizableReadLoop","detail":"{readStateIndex:2676; appliedIndex:2675; }","duration":"860.78601ms","start":"2026-03-19T21:56:12.836069Z","end":"2026-03-19T21:56:13.696855Z","steps":["trace[144247570] 'read index received' (duration: 139.766978ms)","trace[144247570] 'applied index is now lower than readState.Index' (duration: 721.017682ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:56:13.696967Z","caller":"traceutil/trace.go:171","msg":"trace[1605559628] transaction","detail":"{read_only:false; response_revision:2571; number_of_response:1; }","duration":"862.565209ms","start":"2026-03-19T21:56:12.834368Z","end":"2026-03-19T21:56:13.696933Z","steps":["trace[1605559628] 'process raft request' (duration: 141.529966ms)","trace[1605559628] 'compare' (duration: 720.699334ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:13.697057Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"860.994475ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs/auth-system/keycloak\" ","response":"range_response_count:1 size:1319"} {"level":"info","ts":"2026-03-19T21:56:13.697088Z","caller":"traceutil/trace.go:171","msg":"trace[1457383309] range","detail":"{range_begin:/registry/services/specs/auth-system/keycloak; range_end:; response_count:1; response_revision:2571; }","duration":"861.047717ms","start":"2026-03-19T21:56:12.83603Z","end":"2026-03-19T21:56:13.697078Z","steps":["trace[1457383309] 'agreement among raft nodes before linearized reading' (duration: 860.898983ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:13.697132Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:12.836013Z","time spent":"861.110548ms","remote":"127.0.0.1:57040","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":1343,"request content":"key:\"/registry/services/specs/auth-system/keycloak\" "} {"level":"warn","ts":"2026-03-19T21:56:13.697195Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:12.834348Z","time spent":"862.717903ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":491,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:56:13.874998Z","caller":"traceutil/trace.go:171","msg":"trace[636538398] linearizableReadLoop","detail":"{readStateIndex:2677; appliedIndex:2676; }","duration":"178.070446ms","start":"2026-03-19T21:56:13.6969Z","end":"2026-03-19T21:56:13.874971Z","steps":["trace[636538398] 'read index received' (duration: 163.801334ms)","trace[636538398] 'applied index is now lower than readState.Index' (duration: 14.267862ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:13.875766Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"273.966048ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs/ingress-nginx/ingress-nginx-controller\" ","response":"range_response_count:1 size:1990"} {"level":"info","ts":"2026-03-19T21:56:13.875897Z","caller":"traceutil/trace.go:171","msg":"trace[270408172] range","detail":"{range_begin:/registry/services/specs/ingress-nginx/ingress-nginx-controller; range_end:; response_count:1; response_revision:2572; }","duration":"274.125002ms","start":"2026-03-19T21:56:13.601718Z","end":"2026-03-19T21:56:13.875843Z","steps":["trace[270408172] 'agreement among raft nodes before linearized reading' (duration: 273.393741ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:13.875902Z","caller":"traceutil/trace.go:171","msg":"trace[688078223] transaction","detail":"{read_only:false; response_revision:2572; number_of_response:1; }","duration":"192.837602ms","start":"2026-03-19T21:56:13.683012Z","end":"2026-03-19T21:56:13.87585Z","steps":["trace[688078223] 'process raft request' (duration: 177.739406ms)","trace[688078223] 'compare' (duration: 14.092197ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:13.875963Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"669.354496ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" ","response":"range_response_count:1 size:555"} {"level":"warn","ts":"2026-03-19T21:56:13.876007Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"235.726931ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:433"} {"level":"info","ts":"2026-03-19T21:56:13.876065Z","caller":"traceutil/trace.go:171","msg":"trace[338128961] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:2572; }","duration":"235.802723ms","start":"2026-03-19T21:56:13.64025Z","end":"2026-03-19T21:56:13.876053Z","steps":["trace[338128961] 'agreement among raft nodes before linearized reading' (duration: 234.853216ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:13.876084Z","caller":"traceutil/trace.go:171","msg":"trace[1323034397] range","detail":"{range_begin:/registry/leases/openstack/rabbitmq-cluster-operator-leader-election; range_end:; response_count:1; response_revision:2572; }","duration":"669.50097ms","start":"2026-03-19T21:56:13.206531Z","end":"2026-03-19T21:56:13.876032Z","steps":["trace[1323034397] 'agreement among raft nodes before linearized reading' (duration: 669.096788ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:13.876133Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:13.206509Z","time spent":"669.612393ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":70,"response count":1,"response size":579,"request content":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" "} {"level":"warn","ts":"2026-03-19T21:56:13.876103Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"235.709941ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-19T21:56:13.876359Z","caller":"traceutil/trace.go:171","msg":"trace[1352122637] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:2572; }","duration":"236.164394ms","start":"2026-03-19T21:56:13.640099Z","end":"2026-03-19T21:56:13.876263Z","steps":["trace[1352122637] 'agreement among raft nodes before linearized reading' (duration: 234.941959ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:13.876509Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"236.147812ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" ","response":"range_response_count:1 size:479"} {"level":"info","ts":"2026-03-19T21:56:13.876579Z","caller":"traceutil/trace.go:171","msg":"trace[45977478] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:2572; }","duration":"236.226715ms","start":"2026-03-19T21:56:13.640324Z","end":"2026-03-19T21:56:13.876551Z","steps":["trace[45977478] 'agreement among raft nodes before linearized reading' (duration: 235.864125ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:13.877498Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"176.332647ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/statefulsets/auth-system/keycloak\" ","response":"range_response_count:1 size:7460"} {"level":"info","ts":"2026-03-19T21:56:13.877568Z","caller":"traceutil/trace.go:171","msg":"trace[118129915] range","detail":"{range_begin:/registry/statefulsets/auth-system/keycloak; range_end:; response_count:1; response_revision:2572; }","duration":"176.420389ms","start":"2026-03-19T21:56:13.701118Z","end":"2026-03-19T21:56:13.877538Z","steps":["trace[118129915] 'agreement among raft nodes before linearized reading' (duration: 176.09159ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:13.877627Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"237.284915ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" ","response":"range_response_count:1 size:492"} {"level":"info","ts":"2026-03-19T21:56:13.877713Z","caller":"traceutil/trace.go:171","msg":"trace[408765873] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:2572; }","duration":"237.382738ms","start":"2026-03-19T21:56:13.64031Z","end":"2026-03-19T21:56:13.877692Z","steps":["trace[408765873] 'agreement among raft nodes before linearized reading' (duration: 234.707542ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:19.128977Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"122.784359ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.19.213.66\" ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-19T21:56:19.129081Z","caller":"traceutil/trace.go:171","msg":"trace[696652945] range","detail":"{range_begin:/registry/masterleases/199.19.213.66; range_end:; response_count:1; response_revision:2599; }","duration":"122.899892ms","start":"2026-03-19T21:56:19.006158Z","end":"2026-03-19T21:56:19.129058Z","steps":["trace[696652945] 'range keys from in-memory index tree' (duration: 122.599153ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:19.12952Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"290.477374ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/auth-system/keycloak-0.189e5cd082219334\" ","response":"range_response_count:1 size:781"} {"level":"info","ts":"2026-03-19T21:56:19.1297Z","caller":"traceutil/trace.go:171","msg":"trace[1361141250] range","detail":"{range_begin:/registry/events/auth-system/keycloak-0.189e5cd082219334; range_end:; response_count:1; response_revision:2599; }","duration":"290.753391ms","start":"2026-03-19T21:56:18.838921Z","end":"2026-03-19T21:56:19.129674Z","steps":["trace[1361141250] 'range keys from in-memory index tree' (duration: 290.079773ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:19.654698Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"271.769556ms","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:41"} {"level":"warn","ts":"2026-03-19T21:56:19.655122Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.130126Z","time spent":"524.988369ms","remote":"127.0.0.1:56898","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-19T21:56:19.65536Z","caller":"traceutil/trace.go:171","msg":"trace[1861574867] transaction","detail":"{read_only:false; response_revision:2601; number_of_response:1; }","duration":"522.769557ms","start":"2026-03-19T21:56:19.132571Z","end":"2026-03-19T21:56:19.65534Z","steps":["trace[1861574867] 'process raft request' (duration: 522.400377ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:19.655351Z","caller":"traceutil/trace.go:171","msg":"trace[1441162069] linearizableReadLoop","detail":"{readStateIndex:2708; appliedIndex:2705; }","duration":"512.772334ms","start":"2026-03-19T21:56:19.142555Z","end":"2026-03-19T21:56:19.655327Z","steps":["trace[1441162069] 'read index received' (duration: 50.864702ms)","trace[1441162069] 'applied index is now lower than readState.Index' (duration: 461.905522ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:19.655577Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.132547Z","time spent":"522.88624ms","remote":"127.0.0.1:56936","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":766,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:56:19.655698Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"513.117475ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" ","response":"range_response_count:1 size:12215"} {"level":"info","ts":"2026-03-19T21:56:19.65585Z","caller":"traceutil/trace.go:171","msg":"trace[758787229] range","detail":"{range_begin:/registry/pods/openstack/percona-xtradb-pxc-0; range_end:; response_count:1; response_revision:2601; }","duration":"513.31273ms","start":"2026-03-19T21:56:19.142513Z","end":"2026-03-19T21:56:19.655826Z","steps":["trace[758787229] 'agreement among raft nodes before linearized reading' (duration: 512.916219ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:19.655905Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.1425Z","time spent":"513.392992ms","remote":"127.0.0.1:57034","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":12239,"request content":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" "} {"level":"warn","ts":"2026-03-19T21:56:20.160454Z","caller":"etcdserver/v3_server.go:897","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8713512037097302046,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-19T21:56:21.986635Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000183585s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-19T21:56:21.986726Z","caller":"traceutil/trace.go:171","msg":"trace[310186062] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.00033816s","start":"2026-03-19T21:56:19.986372Z","end":"2026-03-19T21:56:21.98671Z","steps":["trace[310186062] 'agreement among raft nodes before linearized reading' (duration: 2.000225587s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:21.986778Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.98635Z","time spent":"2.000415242s","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/19 21:56:21 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-19T21:56:22.013483Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.534152777s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:56:22.013599Z","caller":"traceutil/trace.go:171","msg":"trace[222619438] transaction","detail":"{read_only:false; response_revision:2602; number_of_response:1; }","duration":"2.357609004s","start":"2026-03-19T21:56:19.655968Z","end":"2026-03-19T21:56:22.013577Z","steps":["trace[222619438] 'process raft request' (duration: 823.258942ms)","trace[222619438] 'compare' (duration: 1.533990332s)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:22.01367Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.655946Z","time spent":"2.357693976s","remote":"127.0.0.1:56898","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":118,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:56:22.253438Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.000615718s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-19T21:56:22.253623Z","caller":"traceutil/trace.go:171","msg":"trace[800988773] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000862295s","start":"2026-03-19T21:56:20.252734Z","end":"2026-03-19T21:56:22.253596Z","steps":["trace[800988773] 'agreement among raft nodes before linearized reading' (duration: 2.000650099s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:22.253685Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:20.252714Z","time spent":"2.000958608s","remote":"127.0.0.1:56892","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/19 21:56:22 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-19T21:56:22.614957Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"1.504935383s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-19T21:56:23.192245Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"576.985183ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-19T21:56:23.192507Z","caller":"traceutil/trace.go:171","msg":"trace[1550279245] transaction","detail":"{read_only:false; response_revision:2604; number_of_response:1; }","duration":"2.675722699s","start":"2026-03-19T21:56:20.516759Z","end":"2026-03-19T21:56:23.192481Z","steps":["trace[1550279245] 'process raft request' (duration: 2.675633057s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.192614Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:20.51672Z","time spent":"2.675844041s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":509,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T21:56:23.192792Z","caller":"traceutil/trace.go:171","msg":"trace[180529680] linearizableReadLoop","detail":"{readStateIndex:2710; appliedIndex:2708; }","duration":"3.532832592s","start":"2026-03-19T21:56:19.659924Z","end":"2026-03-19T21:56:23.192756Z","steps":["trace[180529680] 'read index received' (duration: 819.321102ms)","trace[180529680] 'applied index is now lower than readState.Index' (duration: 2.71350835s)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:56:23.192795Z","caller":"traceutil/trace.go:171","msg":"trace[1759655347] transaction","detail":"{read_only:false; response_revision:2603; number_of_response:1; }","duration":"3.285848254s","start":"2026-03-19T21:56:19.906912Z","end":"2026-03-19T21:56:23.19276Z","steps":["trace[1759655347] 'process raft request' (duration: 2.708198308s)","trace[1759655347] 'compare' (duration: 576.587336ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T21:56:23.193088Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.17717194s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" ","response":"range_response_count:1 size:421"} {"level":"warn","ts":"2026-03-19T21:56:23.19308Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.533134767s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" ","response":"range_response_count:1 size:12215"} {"level":"info","ts":"2026-03-19T21:56:23.193169Z","caller":"traceutil/trace.go:171","msg":"trace[228000582] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:2604; }","duration":"1.177265812s","start":"2026-03-19T21:56:22.015872Z","end":"2026-03-19T21:56:23.193138Z","steps":["trace[228000582] 'agreement among raft nodes before linearized reading' (duration: 1.177127119s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.193204Z","caller":"traceutil/trace.go:171","msg":"trace[1530778356] range","detail":"{range_begin:/registry/pods/openstack/percona-xtradb-pxc-0; range_end:; response_count:1; response_revision:2604; }","duration":"3.533302292s","start":"2026-03-19T21:56:19.659877Z","end":"2026-03-19T21:56:23.193179Z","steps":["trace[1530778356] 'agreement among raft nodes before linearized reading' (duration: 3.532967465s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.193214Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.649620828s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs/auth-system/keycloak-headless\" ","response":"range_response_count:1 size:1369"} {"level":"warn","ts":"2026-03-19T21:56:23.193281Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.659862Z","time spent":"3.533404274s","remote":"127.0.0.1:57034","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":12239,"request content":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193267Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.203446728s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-instance.189e5cbb1f902186\" ","response":"range_response_count:1 size:773"} {"level":"warn","ts":"2026-03-19T21:56:23.193319Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.194149596s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" ","response":"range_response_count:1 size:480"} {"level":"warn","ts":"2026-03-19T21:56:23.193345Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.241765788s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" ","response":"range_response_count:1 size:438"} {"level":"warn","ts":"2026-03-19T21:56:23.193367Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.194233918s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:434"} {"level":"info","ts":"2026-03-19T21:56:23.193381Z","caller":"traceutil/trace.go:171","msg":"trace[618137090] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:2604; }","duration":"3.194246778s","start":"2026-03-19T21:56:19.999116Z","end":"2026-03-19T21:56:23.193363Z","steps":["trace[618137090] 'agreement among raft nodes before linearized reading' (duration: 3.194041915s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.193292Z","caller":"traceutil/trace.go:171","msg":"trace[975110202] range","detail":"{range_begin:/registry/services/specs/auth-system/keycloak-headless; range_end:; response_count:1; response_revision:2604; }","duration":"2.649760521s","start":"2026-03-19T21:56:20.54351Z","end":"2026-03-19T21:56:23.19327Z","steps":["trace[975110202] 'agreement among raft nodes before linearized reading' (duration: 2.649475534s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.193379Z","caller":"traceutil/trace.go:171","msg":"trace[1224824926] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-instance.189e5cbb1f902186; range_end:; response_count:1; response_revision:2604; }","duration":"1.203589551s","start":"2026-03-19T21:56:21.989757Z","end":"2026-03-19T21:56:23.193346Z","steps":["trace[1224824926] 'agreement among raft nodes before linearized reading' (duration: 1.203270454s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.19341Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"756.002735ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csistoragecapacities/\" range_end:\"/registry/csistoragecapacities0\" count_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:56:23.193412Z","caller":"traceutil/trace.go:171","msg":"trace[1093028815] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:2604; }","duration":"3.24185833s","start":"2026-03-19T21:56:19.951527Z","end":"2026-03-19T21:56:23.193385Z","steps":["trace[1093028815] 'agreement among raft nodes before linearized reading' (duration: 3.241695347s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.193419Z","caller":"traceutil/trace.go:171","msg":"trace[1629277564] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:2604; }","duration":"3.19431114s","start":"2026-03-19T21:56:19.999094Z","end":"2026-03-19T21:56:23.193405Z","steps":["trace[1629277564] 'agreement among raft nodes before linearized reading' (duration: 3.194124267s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.193414Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.193045874s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" ","response":"range_response_count:1 size:555"} {"level":"warn","ts":"2026-03-19T21:56:23.193463Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.951507Z","time spent":"3.241946232s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":462,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193465Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.999079Z","time spent":"3.194378301s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":44,"response count":1,"response size":458,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193482Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:20.54349Z","time spent":"2.649943314s","remote":"127.0.0.1:57040","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":1393,"request content":"key:\"/registry/services/specs/auth-system/keycloak-headless\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193305Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:22.015861Z","time spent":"1.177395414s","remote":"127.0.0.1:57012","response type":"/etcdserverpb.KV/Range","request count":0,"request size":49,"response count":1,"response size":445,"request content":"key:\"/registry/services/endpoints/default/kubernetes\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193486Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"932.307657ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-19T21:56:23.193462Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:21.989742Z","time spent":"1.203700674s","remote":"127.0.0.1:56936","response type":"/etcdserverpb.KV/Range","request count":0,"request size":71,"response count":1,"response size":797,"request content":"key:\"/registry/events/kube-system/kube-apiserver-instance.189e5cbb1f902186\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193425Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.999098Z","time spent":"3.19431662s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":45,"response count":1,"response size":504,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" "} {"level":"warn","ts":"2026-03-19T21:56:23.19357Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.19368087s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" ","response":"range_response_count:1 size:492"} {"level":"warn","ts":"2026-03-19T21:56:23.193135Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.236395927s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/\" range_end:\"/registry/pxc.percona.com/perconaxtradbclusters0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-03-19T21:56:23.193621Z","caller":"traceutil/trace.go:171","msg":"trace[973989653] range","detail":"{range_begin:/registry/leases/kube-system/cilium-operator-resource-lock; range_end:; response_count:1; response_revision:2604; }","duration":"3.193754672s","start":"2026-03-19T21:56:19.999853Z","end":"2026-03-19T21:56:23.193607Z","steps":["trace[973989653] 'agreement among raft nodes before linearized reading' (duration: 3.193591759s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.193599Z","caller":"traceutil/trace.go:171","msg":"trace[1460949182] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:2604; }","duration":"932.511562ms","start":"2026-03-19T21:56:22.261069Z","end":"2026-03-19T21:56:23.19358Z","steps":["trace[1460949182] 'agreement among raft nodes before linearized reading' (duration: 932.288997ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.193647Z","caller":"traceutil/trace.go:171","msg":"trace[1296267259] range","detail":"{range_begin:/registry/pxc.percona.com/perconaxtradbclusters/; range_end:/registry/pxc.percona.com/perconaxtradbclusters0; response_count:0; response_revision:2604; }","duration":"1.236970449s","start":"2026-03-19T21:56:21.956652Z","end":"2026-03-19T21:56:23.193622Z","steps":["trace[1296267259] 'agreement among raft nodes before linearized reading' (duration: 1.236393497s)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.193677Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.999843Z","time spent":"3.193822183s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":60,"response count":1,"response size":516,"request content":"key:\"/registry/leases/kube-system/cilium-operator-resource-lock\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193687Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:22.261047Z","time spent":"932.623893ms","remote":"127.0.0.1:56878","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":29,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193725Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:21.956633Z","time spent":"1.237057011s","remote":"127.0.0.1:51184","response type":"/etcdserverpb.KV/Range","request count":0,"request size":102,"response count":1,"response size":31,"request content":"key:\"/registry/pxc.percona.com/perconaxtradbclusters/\" range_end:\"/registry/pxc.percona.com/perconaxtradbclusters0\" count_only:true "} {"level":"warn","ts":"2026-03-19T21:56:23.193184Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.439731616s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" ","response":"range_response_count:1 size:507"} {"level":"info","ts":"2026-03-19T21:56:23.193487Z","caller":"traceutil/trace.go:171","msg":"trace[277693568] range","detail":"{range_begin:/registry/leases/openstack/rabbitmq-cluster-operator-leader-election; range_end:; response_count:1; response_revision:2604; }","duration":"3.193136026s","start":"2026-03-19T21:56:20.00033Z","end":"2026-03-19T21:56:23.193466Z","steps":["trace[277693568] 'agreement among raft nodes before linearized reading' (duration: 3.192883122s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.193825Z","caller":"traceutil/trace.go:171","msg":"trace[1632683791] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:2604; }","duration":"3.44047186s","start":"2026-03-19T21:56:19.753342Z","end":"2026-03-19T21:56:23.193813Z","steps":["trace[1632683791] 'agreement among raft nodes before linearized reading' (duration: 3.439634053s)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.193461Z","caller":"traceutil/trace.go:171","msg":"trace[1026995348] range","detail":"{range_begin:/registry/csistoragecapacities/; range_end:/registry/csistoragecapacities0; response_count:0; response_revision:2604; }","duration":"756.099937ms","start":"2026-03-19T21:56:22.437343Z","end":"2026-03-19T21:56:23.193443Z","steps":["trace[1026995348] 'agreement among raft nodes before linearized reading' (duration: 755.982805ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.193851Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:20.000321Z","time spent":"3.193517003s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":70,"response count":1,"response size":579,"request content":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" "} {"level":"warn","ts":"2026-03-19T21:56:23.193875Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.753319Z","time spent":"3.440544132s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Range","request count":0,"request size":54,"response count":1,"response size":531,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" "} {"level":"warn","ts":"2026-03-19T21:56:23.19368Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:19.90689Z","time spent":"3.286039548s","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":555,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-19T21:56:23.193941Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T21:56:22.43732Z","time spent":"756.566616ms","remote":"127.0.0.1:57248","response type":"/etcdserverpb.KV/Range","request count":0,"request size":68,"response count":0,"response size":29,"request content":"key:\"/registry/csistoragecapacities/\" range_end:\"/registry/csistoragecapacities0\" count_only:true "} {"level":"info","ts":"2026-03-19T21:56:23.41763Z","caller":"traceutil/trace.go:171","msg":"trace[229864019] linearizableReadLoop","detail":"{readStateIndex:2718; appliedIndex:2712; }","duration":"204.066929ms","start":"2026-03-19T21:56:23.213538Z","end":"2026-03-19T21:56:23.417605Z","steps":["trace[229864019] 'read index received' (duration: 194.507106ms)","trace[229864019] 'applied index is now lower than readState.Index' (duration: 9.558543ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T21:56:23.417804Z","caller":"traceutil/trace.go:171","msg":"trace[660123855] transaction","detail":"{read_only:false; response_revision:2611; number_of_response:1; }","duration":"211.546554ms","start":"2026-03-19T21:56:23.206224Z","end":"2026-03-19T21:56:23.417771Z","steps":["trace[660123855] 'process raft request' (duration: 211.3456ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.417954Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"217.163611ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-19T21:56:23.417964Z","caller":"traceutil/trace.go:171","msg":"trace[1948599336] transaction","detail":"{read_only:false; response_revision:2609; number_of_response:1; }","duration":"216.893336ms","start":"2026-03-19T21:56:23.201048Z","end":"2026-03-19T21:56:23.417941Z","steps":["trace[1948599336] 'process raft request' (duration: 216.455018ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.417995Z","caller":"traceutil/trace.go:171","msg":"trace[478518465] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:1; response_revision:2611; }","duration":"217.369345ms","start":"2026-03-19T21:56:23.200616Z","end":"2026-03-19T21:56:23.417986Z","steps":["trace[478518465] 'agreement among raft nodes before linearized reading' (duration: 217.12608ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.417979Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"154.96139ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/deployments/openstack/percona-xtradb-pitr\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T21:56:23.417981Z","caller":"traceutil/trace.go:171","msg":"trace[1342100439] transaction","detail":"{read_only:false; response_revision:2608; number_of_response:1; }","duration":"218.121499ms","start":"2026-03-19T21:56:23.199599Z","end":"2026-03-19T21:56:23.417721Z","steps":["trace[1342100439] 'process raft request' (duration: 217.870874ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.418112Z","caller":"traceutil/trace.go:171","msg":"trace[449995674] transaction","detail":"{read_only:false; response_revision:2610; number_of_response:1; }","duration":"216.819184ms","start":"2026-03-19T21:56:23.201279Z","end":"2026-03-19T21:56:23.418098Z","steps":["trace[449995674] 'process raft request' (duration: 216.262893ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T21:56:23.418134Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"201.101603ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-instance.189e5cbb2f57e45e\" ","response":"range_response_count:1 size:774"} {"level":"warn","ts":"2026-03-19T21:56:23.418196Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"201.110754ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs/auth-system/keycloak\" ","response":"range_response_count:1 size:1319"} {"level":"info","ts":"2026-03-19T21:56:23.418222Z","caller":"traceutil/trace.go:171","msg":"trace[605489167] range","detail":"{range_begin:/registry/services/specs/auth-system/keycloak; range_end:; response_count:1; response_revision:2611; }","duration":"201.178214ms","start":"2026-03-19T21:56:23.217038Z","end":"2026-03-19T21:56:23.418216Z","steps":["trace[605489167] 'agreement among raft nodes before linearized reading' (duration: 201.042281ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.418256Z","caller":"traceutil/trace.go:171","msg":"trace[1361335833] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-instance.189e5cbb2f57e45e; range_end:; response_count:1; response_revision:2611; }","duration":"201.249546ms","start":"2026-03-19T21:56:23.216965Z","end":"2026-03-19T21:56:23.418215Z","steps":["trace[1361335833] 'agreement among raft nodes before linearized reading' (duration: 200.899189ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.418269Z","caller":"traceutil/trace.go:171","msg":"trace[693208658] transaction","detail":"{read_only:false; response_revision:2606; number_of_response:1; }","duration":"219.105378ms","start":"2026-03-19T21:56:23.199151Z","end":"2026-03-19T21:56:23.418257Z","steps":["trace[693208658] 'process raft request' (duration: 208.961663ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.41805Z","caller":"traceutil/trace.go:171","msg":"trace[1833772896] range","detail":"{range_begin:/registry/deployments/openstack/percona-xtradb-pitr; range_end:; response_count:0; response_revision:2611; }","duration":"155.092812ms","start":"2026-03-19T21:56:23.262939Z","end":"2026-03-19T21:56:23.418032Z","steps":["trace[1833772896] 'agreement among raft nodes before linearized reading' (duration: 154.945659ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T21:56:23.419761Z","caller":"traceutil/trace.go:171","msg":"trace[41999007] transaction","detail":"{read_only:false; response_revision:2607; number_of_response:1; }","duration":"218.844972ms","start":"2026-03-19T21:56:23.199535Z","end":"2026-03-19T21:56:23.41838Z","steps":["trace[41999007] 'process raft request' (duration: 217.895854ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:00:24.09077Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"154.985359ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets/monitoring/prometheus-kube-prometheus-stack-prometheus\" ","response":"range_response_count:1 size:3506"} {"level":"info","ts":"2026-03-19T22:00:24.09089Z","caller":"traceutil/trace.go:171","msg":"trace[227993123] range","detail":"{range_begin:/registry/secrets/monitoring/prometheus-kube-prometheus-stack-prometheus; range_end:; response_count:1; response_revision:4493; }","duration":"155.166863ms","start":"2026-03-19T22:00:23.935697Z","end":"2026-03-19T22:00:24.090864Z","steps":["trace[227993123] 'range keys from in-memory index tree' (duration: 154.740132ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:00:26.717385Z","caller":"traceutil/trace.go:171","msg":"trace[2131380024] transaction","detail":"{read_only:false; response_revision:4549; number_of_response:1; }","duration":"148.917041ms","start":"2026-03-19T22:00:26.568415Z","end":"2026-03-19T22:00:26.717332Z","steps":["trace[2131380024] 'process raft request' (duration: 148.643984ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:00:29.201659Z","caller":"traceutil/trace.go:171","msg":"trace[657062586] linearizableReadLoop","detail":"{readStateIndex:4882; appliedIndex:4881; }","duration":"236.044583ms","start":"2026-03-19T22:00:28.965585Z","end":"2026-03-19T22:00:29.20163Z","steps":["trace[657062586] 'read index received' (duration: 144.525212ms)","trace[657062586] 'applied index is now lower than readState.Index' (duration: 91.518661ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T22:00:29.20172Z","caller":"traceutil/trace.go:171","msg":"trace[1854605238] transaction","detail":"{read_only:false; response_revision:4694; number_of_response:1; }","duration":"236.553547ms","start":"2026-03-19T22:00:28.965131Z","end":"2026-03-19T22:00:29.201685Z","steps":["trace[1854605238] 'process raft request' (duration: 145.041986ms)","trace[1854605238] 'compare' (duration: 91.328716ms)"],"step_count":2} {"level":"warn","ts":"2026-03-19T22:00:29.201888Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"236.278059ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/monitoring/vector\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-19T22:00:29.201964Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"100.283451ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/local-path-storage/helper-pod-create-pvc-2ec0bcda-2c12-4155-a49d-96f180b9525d\" ","response":"range_response_count:1 size:3600"} {"level":"info","ts":"2026-03-19T22:00:29.201996Z","caller":"traceutil/trace.go:171","msg":"trace[1515088253] range","detail":"{range_begin:/registry/pods/local-path-storage/helper-pod-create-pvc-2ec0bcda-2c12-4155-a49d-96f180b9525d; range_end:; response_count:1; response_revision:4694; }","duration":"100.388385ms","start":"2026-03-19T22:00:29.101597Z","end":"2026-03-19T22:00:29.201986Z","steps":["trace[1515088253] 'agreement among raft nodes before linearized reading' (duration: 100.230571ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:00:29.201995Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"223.418718ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:434"} {"level":"info","ts":"2026-03-19T22:00:29.202072Z","caller":"traceutil/trace.go:171","msg":"trace[1334043249] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:4694; }","duration":"223.537441ms","start":"2026-03-19T22:00:28.978514Z","end":"2026-03-19T22:00:29.202051Z","steps":["trace[1334043249] 'agreement among raft nodes before linearized reading' (duration: 223.316465ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:00:29.20209Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"181.390759ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.19.213.66\" ","response":"range_response_count:1 size:133"} {"level":"info","ts":"2026-03-19T22:00:29.202167Z","caller":"traceutil/trace.go:171","msg":"trace[1795931326] range","detail":"{range_begin:/registry/serviceaccounts/monitoring/vector; range_end:; response_count:0; response_revision:4694; }","duration":"236.415864ms","start":"2026-03-19T22:00:28.965561Z","end":"2026-03-19T22:00:29.201977Z","steps":["trace[1795931326] 'agreement among raft nodes before linearized reading' (duration: 236.246639ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:00:29.202183Z","caller":"traceutil/trace.go:171","msg":"trace[58130039] range","detail":"{range_begin:/registry/masterleases/199.19.213.66; range_end:; response_count:1; response_revision:4694; }","duration":"181.546403ms","start":"2026-03-19T22:00:29.020614Z","end":"2026-03-19T22:00:29.20216Z","steps":["trace[58130039] 'agreement among raft nodes before linearized reading' (duration: 181.273275ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:00:29.345981Z","caller":"traceutil/trace.go:171","msg":"trace[1973671279] transaction","detail":"{read_only:false; response_revision:4695; number_of_response:1; }","duration":"139.983577ms","start":"2026-03-19T22:00:29.205966Z","end":"2026-03-19T22:00:29.34595Z","steps":["trace[1973671279] 'process raft request' (duration: 139.704269ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:00:29.346172Z","caller":"traceutil/trace.go:171","msg":"trace[133398262] transaction","detail":"{read_only:false; response_revision:4696; number_of_response:1; }","duration":"109.394911ms","start":"2026-03-19T22:00:29.236738Z","end":"2026-03-19T22:00:29.346133Z","steps":["trace[133398262] 'process raft request' (duration: 109.108333ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:00:39.506754Z","caller":"traceutil/trace.go:171","msg":"trace[574077780] transaction","detail":"{read_only:false; response_revision:4885; number_of_response:1; }","duration":"100.235011ms","start":"2026-03-19T22:00:39.406493Z","end":"2026-03-19T22:00:39.506728Z","steps":["trace[574077780] 'process raft request' (duration: 58.733276ms)","trace[574077780] 'compare' (duration: 41.282349ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T22:00:46.476317Z","caller":"traceutil/trace.go:171","msg":"trace[1038465150] transaction","detail":"{read_only:false; response_revision:4949; number_of_response:1; }","duration":"102.669406ms","start":"2026-03-19T22:00:46.373619Z","end":"2026-03-19T22:00:46.476288Z","steps":["trace[1038465150] 'process raft request' (duration: 98.1567ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:01:06.451839Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"268.265381ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rabbitmq.com/rabbitmqclusters/openstack/rabbitmq-keystone\" ","response":"range_response_count:1 size:3243"} {"level":"warn","ts":"2026-03-19T22:01:06.451901Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"624.052775ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/openstack/rabbitmq-keystone-server-0\" ","response":"range_response_count:1 size:10391"} {"level":"info","ts":"2026-03-19T22:01:06.451997Z","caller":"traceutil/trace.go:171","msg":"trace[831287693] range","detail":"{range_begin:/registry/pods/openstack/rabbitmq-keystone-server-0; range_end:; response_count:1; response_revision:5242; }","duration":"624.19814ms","start":"2026-03-19T22:01:05.827774Z","end":"2026-03-19T22:01:06.451972Z","steps":["trace[831287693] 'range keys from in-memory index tree' (duration: 623.84008ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:01:06.45201Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"616.385883ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/openstack/keepalived-jfl2n\" ","response":"range_response_count:1 size:6527"} {"level":"warn","ts":"2026-03-19T22:01:06.452053Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T22:01:05.82774Z","time spent":"624.299693ms","remote":"127.0.0.1:57034","response type":"/etcdserverpb.KV/Range","request count":0,"request size":53,"response count":1,"response size":10415,"request content":"key:\"/registry/pods/openstack/rabbitmq-keystone-server-0\" "} {"level":"info","ts":"2026-03-19T22:01:06.452075Z","caller":"traceutil/trace.go:171","msg":"trace[686388459] range","detail":"{range_begin:/registry/pods/openstack/keepalived-jfl2n; range_end:; response_count:1; response_revision:5242; }","duration":"616.502076ms","start":"2026-03-19T22:01:05.835559Z","end":"2026-03-19T22:01:06.452061Z","steps":["trace[686388459] 'range keys from in-memory index tree' (duration: 616.252059ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:01:06.452117Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T22:01:05.835537Z","time spent":"616.562268ms","remote":"127.0.0.1:57034","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":6551,"request content":"key:\"/registry/pods/openstack/keepalived-jfl2n\" "} {"level":"info","ts":"2026-03-19T22:01:06.451966Z","caller":"traceutil/trace.go:171","msg":"trace[445489467] range","detail":"{range_begin:/registry/rabbitmq.com/rabbitmqclusters/openstack/rabbitmq-keystone; range_end:; response_count:1; response_revision:5242; }","duration":"268.432926ms","start":"2026-03-19T22:01:06.183508Z","end":"2026-03-19T22:01:06.45194Z","steps":["trace[445489467] 'range keys from in-memory index tree' (duration: 268.048644ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:01:06.452235Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"604.999006ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumeclaims/openstack/persistence-rabbitmq-keystone-server-0\" ","response":"range_response_count:1 size:1865"} {"level":"info","ts":"2026-03-19T22:01:06.452391Z","caller":"traceutil/trace.go:171","msg":"trace[101889139] range","detail":"{range_begin:/registry/persistentvolumeclaims/openstack/persistence-rabbitmq-keystone-server-0; range_end:; response_count:1; response_revision:5242; }","duration":"605.186321ms","start":"2026-03-19T22:01:05.847182Z","end":"2026-03-19T22:01:06.452368Z","steps":["trace[101889139] 'range keys from in-memory index tree' (duration: 604.751189ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:01:06.452443Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T22:01:05.847162Z","time spent":"605.264874ms","remote":"127.0.0.1:57002","response type":"/etcdserverpb.KV/Range","request count":0,"request size":83,"response count":1,"response size":1889,"request content":"key:\"/registry/persistentvolumeclaims/openstack/persistence-rabbitmq-keystone-server-0\" "} {"level":"info","ts":"2026-03-19T22:01:06.632977Z","caller":"traceutil/trace.go:171","msg":"trace[744284749] transaction","detail":"{read_only:false; response_revision:5243; number_of_response:1; }","duration":"163.850177ms","start":"2026-03-19T22:01:06.468541Z","end":"2026-03-19T22:01:06.632391Z","steps":["trace[744284749] 'process raft request' (duration: 163.538229ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:01:06.96518Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"100.543666ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/persistentvolumeclaims/openstack/persistence-rabbitmq-keystone-server-0\" ","response":"range_response_count:1 size:1865"} {"level":"info","ts":"2026-03-19T22:01:06.965279Z","caller":"traceutil/trace.go:171","msg":"trace[199991919] range","detail":"{range_begin:/registry/persistentvolumeclaims/openstack/persistence-rabbitmq-keystone-server-0; range_end:; response_count:1; response_revision:5249; }","duration":"100.69033ms","start":"2026-03-19T22:01:06.864565Z","end":"2026-03-19T22:01:06.965255Z","steps":["trace[199991919] 'range keys from in-memory index tree' (duration: 100.042611ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:01:07.435507Z","caller":"traceutil/trace.go:171","msg":"trace[1309802187] transaction","detail":"{read_only:false; response_revision:5260; number_of_response:1; }","duration":"104.133886ms","start":"2026-03-19T22:01:07.331331Z","end":"2026-03-19T22:01:07.435465Z","steps":["trace[1309802187] 'process raft request' (duration: 43.058528ms)","trace[1309802187] 'compare' (duration: 60.902703ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T22:01:45.873969Z","caller":"mvcc/index.go:214","msg":"compact tree index","revision":2725} {"level":"info","ts":"2026-03-19T22:01:45.934733Z","caller":"mvcc/kvstore_compaction.go:68","msg":"finished scheduled compaction","compact-revision":2725,"took":"59.313869ms","hash":209698521,"current-db-size-bytes":40370176,"current-db-size":"40 MB","current-db-size-in-use-bytes":30068736,"current-db-size-in-use":"30 MB"} {"level":"info","ts":"2026-03-19T22:01:45.934812Z","caller":"mvcc/hash.go:137","msg":"storing new hash","hash":209698521,"revision":2725,"compact-revision":-1} {"level":"info","ts":"2026-03-19T22:01:49.855577Z","caller":"traceutil/trace.go:171","msg":"trace[100716767] transaction","detail":"{read_only:false; response_revision:5624; number_of_response:1; }","duration":"113.658691ms","start":"2026-03-19T22:01:49.741889Z","end":"2026-03-19T22:01:49.855548Z","steps":["trace[100716767] 'process raft request' (duration: 113.415585ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:01:50.168393Z","caller":"traceutil/trace.go:171","msg":"trace[192620145] linearizableReadLoop","detail":"{readStateIndex:5841; appliedIndex:5840; }","duration":"212.630044ms","start":"2026-03-19T22:01:49.955732Z","end":"2026-03-19T22:01:50.168362Z","steps":["trace[192620145] 'read index received' (duration: 212.451059ms)","trace[192620145] 'applied index is now lower than readState.Index' (duration: 178.265µs)"],"step_count":2} {"level":"info","ts":"2026-03-19T22:01:50.168415Z","caller":"traceutil/trace.go:171","msg":"trace[1275498402] transaction","detail":"{read_only:false; response_revision:5625; number_of_response:1; }","duration":"231.851349ms","start":"2026-03-19T22:01:49.936545Z","end":"2026-03-19T22:01:50.168396Z","steps":["trace[1275498402] 'process raft request' (duration: 231.687664ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:01:50.168567Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"212.82647ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/openstack/percona-xtradb-pxc-0\" ","response":"range_response_count:1 size:12215"} {"level":"info","ts":"2026-03-19T22:01:50.168604Z","caller":"traceutil/trace.go:171","msg":"trace[54950551] range","detail":"{range_begin:/registry/pods/openstack/percona-xtradb-pxc-0; range_end:; response_count:1; response_revision:5625; }","duration":"212.900351ms","start":"2026-03-19T22:01:49.955687Z","end":"2026-03-19T22:01:50.168587Z","steps":["trace[54950551] 'agreement among raft nodes before linearized reading' (duration: 212.764677ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:01:50.168971Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"181.57ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-19T22:01:50.169011Z","caller":"traceutil/trace.go:171","msg":"trace[642810715] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:5625; }","duration":"181.645492ms","start":"2026-03-19T22:01:49.987351Z","end":"2026-03-19T22:01:50.168996Z","steps":["trace[642810715] 'agreement among raft nodes before linearized reading' (duration: 181.598641ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:06:45.915319Z","caller":"mvcc/index.go:214","msg":"compact tree index","revision":5603} {"level":"info","ts":"2026-03-19T22:06:46.043917Z","caller":"mvcc/kvstore_compaction.go:68","msg":"finished scheduled compaction","compact-revision":5603,"took":"127.279407ms","hash":3036419804,"current-db-size-bytes":40370176,"current-db-size":"40 MB","current-db-size-in-use-bytes":21884928,"current-db-size-in-use":"22 MB"} {"level":"info","ts":"2026-03-19T22:06:46.043975Z","caller":"mvcc/hash.go:137","msg":"storing new hash","hash":3036419804,"revision":5603,"compact-revision":2725} {"level":"info","ts":"2026-03-19T22:07:04.064604Z","caller":"traceutil/trace.go:171","msg":"trace[1638484986] transaction","detail":"{read_only:false; response_revision:7790; number_of_response:1; }","duration":"323.481231ms","start":"2026-03-19T22:07:03.741098Z","end":"2026-03-19T22:07:04.064579Z","steps":["trace[1638484986] 'process raft request' (duration: 323.325947ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:07:04.064753Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2026-03-19T22:07:03.74107Z","time spent":"323.612875ms","remote":"127.0.0.1:57090","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":418,"response count":0,"response size":40,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-19T22:07:05.631483Z","caller":"traceutil/trace.go:171","msg":"trace[650753199] transaction","detail":"{read_only:false; response_revision:7798; number_of_response:1; }","duration":"185.808425ms","start":"2026-03-19T22:07:05.445658Z","end":"2026-03-19T22:07:05.631467Z","steps":["trace[650753199] 'process raft request' (duration: 185.697002ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:07:34.345686Z","caller":"traceutil/trace.go:171","msg":"trace[679542876] linearizableReadLoop","detail":"{readStateIndex:8293; appliedIndex:8292; }","duration":"126.682746ms","start":"2026-03-19T22:07:34.218978Z","end":"2026-03-19T22:07:34.345661Z","steps":["trace[679542876] 'read index received' (duration: 126.620744ms)","trace[679542876] 'applied index is now lower than readState.Index' (duration: 60.862µs)"],"step_count":2} {"level":"info","ts":"2026-03-19T22:07:34.345821Z","caller":"traceutil/trace.go:171","msg":"trace[865447531] transaction","detail":"{read_only:false; response_revision:7993; number_of_response:1; }","duration":"150.376718ms","start":"2026-03-19T22:07:34.195401Z","end":"2026-03-19T22:07:34.345778Z","steps":["trace[865447531] 'process raft request' (duration: 150.105741ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:07:34.345964Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"126.924133ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" ","response":"range_response_count:1 size:433"} {"level":"info","ts":"2026-03-19T22:07:34.346023Z","caller":"traceutil/trace.go:171","msg":"trace[766983139] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:7993; }","duration":"127.105007ms","start":"2026-03-19T22:07:34.218907Z","end":"2026-03-19T22:07:34.346012Z","steps":["trace[766983139] 'agreement among raft nodes before linearized reading' (duration: 126.875701ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:07:34.446662Z","caller":"traceutil/trace.go:171","msg":"trace[12606090] transaction","detail":"{read_only:false; response_revision:7994; number_of_response:1; }","duration":"116.804704ms","start":"2026-03-19T22:07:34.329832Z","end":"2026-03-19T22:07:34.446637Z","steps":["trace[12606090] 'process raft request' (duration: 113.77111ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:07:46.832273Z","caller":"traceutil/trace.go:171","msg":"trace[148936534] transaction","detail":"{read_only:false; response_revision:8091; number_of_response:1; }","duration":"217.570929ms","start":"2026-03-19T22:07:46.614669Z","end":"2026-03-19T22:07:46.83224Z","steps":["trace[148936534] 'process raft request' (duration: 217.27424ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:07:46.832416Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"180.471718ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/monitoring.coreos.com/podmonitors/\" range_end:\"/registry/monitoring.coreos.com/podmonitors0\" count_only:true ","response":"range_response_count:0 size:7"} {"level":"info","ts":"2026-03-19T22:07:46.832503Z","caller":"traceutil/trace.go:171","msg":"trace[1538515377] range","detail":"{range_begin:/registry/monitoring.coreos.com/podmonitors/; range_end:/registry/monitoring.coreos.com/podmonitors0; response_count:0; response_revision:8091; }","duration":"180.57084ms","start":"2026-03-19T22:07:46.651913Z","end":"2026-03-19T22:07:46.832484Z","steps":["trace[1538515377] 'agreement among raft nodes before linearized reading' (duration: 180.428077ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:07:46.832749Z","caller":"traceutil/trace.go:171","msg":"trace[1978323159] linearizableReadLoop","detail":"{readStateIndex:8397; appliedIndex:8396; }","duration":"180.22289ms","start":"2026-03-19T22:07:46.651931Z","end":"2026-03-19T22:07:46.832154Z","steps":["trace[1978323159] 'read index received' (duration: 180.003984ms)","trace[1978323159] 'applied index is now lower than readState.Index' (duration: 217.766µs)"],"step_count":2} {"level":"info","ts":"2026-03-19T22:07:46.950675Z","caller":"traceutil/trace.go:171","msg":"trace[2044670236] transaction","detail":"{read_only:false; response_revision:8093; number_of_response:1; }","duration":"104.732234ms","start":"2026-03-19T22:07:46.84592Z","end":"2026-03-19T22:07:46.950652Z","steps":["trace[2044670236] 'process raft request' (duration: 104.59892ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:07:48.005498Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"152.574209ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/openstack/rabbitmq-cluster-operator-leader-election\" ","response":"range_response_count:1 size:556"} {"level":"info","ts":"2026-03-19T22:07:48.005606Z","caller":"traceutil/trace.go:171","msg":"trace[1172517756] range","detail":"{range_begin:/registry/leases/openstack/rabbitmq-cluster-operator-leader-election; range_end:; response_count:1; response_revision:8100; }","duration":"152.755003ms","start":"2026-03-19T22:07:47.852829Z","end":"2026-03-19T22:07:48.005584Z","steps":["trace[1172517756] 'range keys from in-memory index tree' (duration: 152.293861ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:07:48.005848Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"132.744683ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/ceph.rook.io/cephclusters/openstack/ceph\" ","response":"range_response_count:1 size:5146"} {"level":"info","ts":"2026-03-19T22:07:48.006048Z","caller":"traceutil/trace.go:171","msg":"trace[12234451] range","detail":"{range_begin:/registry/ceph.rook.io/cephclusters/openstack/ceph; range_end:; response_count:1; response_revision:8100; }","duration":"132.892118ms","start":"2026-03-19T22:07:47.873035Z","end":"2026-03-19T22:07:48.005927Z","steps":["trace[12234451] 'range keys from in-memory index tree' (duration: 132.318771ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:07:48.222495Z","caller":"traceutil/trace.go:171","msg":"trace[790819361] transaction","detail":"{read_only:false; response_revision:8102; number_of_response:1; }","duration":"201.556807ms","start":"2026-03-19T22:07:48.020918Z","end":"2026-03-19T22:07:48.222474Z","steps":["trace[790819361] 'process raft request' (duration: 201.448684ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:08:57.793252Z","caller":"traceutil/trace.go:171","msg":"trace[1227592071] transaction","detail":"{read_only:false; response_revision:8779; number_of_response:1; }","duration":"137.206124ms","start":"2026-03-19T22:08:57.656018Z","end":"2026-03-19T22:08:57.793224Z","steps":["trace[1227592071] 'process raft request' (duration: 137.03595ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:08:57.877892Z","caller":"traceutil/trace.go:171","msg":"trace[75619363] linearizableReadLoop","detail":"{readStateIndex:9104; appliedIndex:9102; }","duration":"171.691051ms","start":"2026-03-19T22:08:57.706167Z","end":"2026-03-19T22:08:57.877858Z","steps":["trace[75619363] 'read index received' (duration: 86.90874ms)","trace[75619363] 'applied index is now lower than readState.Index' (duration: 84.780571ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T22:08:57.877995Z","caller":"traceutil/trace.go:171","msg":"trace[1751783044] transaction","detail":"{read_only:false; response_revision:8780; number_of_response:1; }","duration":"221.179423ms","start":"2026-03-19T22:08:57.656786Z","end":"2026-03-19T22:08:57.877965Z","steps":["trace[1751783044] 'process raft request' (duration: 220.811953ms)"],"step_count":1} {"level":"warn","ts":"2026-03-19T22:08:57.878188Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"171.99476ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/cert-manager/cert-manager-controller\" ","response":"range_response_count:1 size:529"} {"level":"info","ts":"2026-03-19T22:08:57.878375Z","caller":"traceutil/trace.go:171","msg":"trace[9650801] range","detail":"{range_begin:/registry/leases/cert-manager/cert-manager-controller; range_end:; response_count:1; response_revision:8780; }","duration":"172.153144ms","start":"2026-03-19T22:08:57.706138Z","end":"2026-03-19T22:08:57.878291Z","steps":["trace[9650801] 'agreement among raft nodes before linearized reading' (duration: 171.868635ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:08:58.026601Z","caller":"traceutil/trace.go:171","msg":"trace[516178055] transaction","detail":"{read_only:false; response_revision:8781; number_of_response:1; }","duration":"143.676474ms","start":"2026-03-19T22:08:57.882876Z","end":"2026-03-19T22:08:58.026553Z","steps":["trace[516178055] 'process raft request' (duration: 72.389348ms)","trace[516178055] 'compare' (duration: 70.971307ms)"],"step_count":2} {"level":"info","ts":"2026-03-19T22:08:58.026972Z","caller":"traceutil/trace.go:171","msg":"trace[2130268855] transaction","detail":"{read_only:false; response_revision:8783; number_of_response:1; }","duration":"143.708026ms","start":"2026-03-19T22:08:57.883202Z","end":"2026-03-19T22:08:58.02691Z","steps":["trace[2130268855] 'process raft request' (duration: 143.600233ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:08:58.027027Z","caller":"traceutil/trace.go:171","msg":"trace[1496251444] transaction","detail":"{read_only:false; response_revision:8782; number_of_response:1; }","duration":"143.832159ms","start":"2026-03-19T22:08:57.883163Z","end":"2026-03-19T22:08:58.026995Z","steps":["trace[1496251444] 'process raft request' (duration: 143.328904ms)"],"step_count":1} {"level":"info","ts":"2026-03-19T22:11:08.108919Z","caller":"etcdserver/server.go:1401","msg":"triggering snapshot","local-member-id":"dc35f804a62878ec","local-member-applied-index":10003,"local-member-snapshot-index":0,"local-member-snapshot-count":10000} {"level":"info","ts":"2026-03-19T22:11:08.116952Z","caller":"etcdserver/server.go:2420","msg":"saved snapshot","snapshot-index":10003} {"level":"info","ts":"2026-03-19T22:11:08.117044Z","caller":"etcdserver/server.go:2450","msg":"compacted Raft logs","compact-index":5003} {"level":"info","ts":"2026-03-19T22:11:45.923108Z","caller":"mvcc/index.go:214","msg":"compact tree index","revision":7574} {"level":"info","ts":"2026-03-19T22:11:45.98618Z","caller":"mvcc/kvstore_compaction.go:68","msg":"finished scheduled compaction","compact-revision":7574,"took":"61.200373ms","hash":2343597325,"current-db-size-bytes":40370176,"current-db-size":"40 MB","current-db-size-in-use-bytes":26095616,"current-db-size-in-use":"26 MB"} {"level":"info","ts":"2026-03-19T22:11:45.986255Z","caller":"mvcc/hash.go:137","msg":"storing new hash","hash":2343597325,"revision":7574,"compact-revision":5603} {"level":"info","ts":"2026-03-19T22:16:45.932604Z","caller":"mvcc/index.go:214","msg":"compact tree index","revision":9952} {"level":"info","ts":"2026-03-19T22:16:46.014132Z","caller":"mvcc/kvstore_compaction.go:68","msg":"finished scheduled compaction","compact-revision":9952,"took":"79.280375ms","hash":1601956741,"current-db-size-bytes":40370176,"current-db-size":"40 MB","current-db-size-in-use-bytes":17350656,"current-db-size-in-use":"17 MB"} {"level":"info","ts":"2026-03-19T22:16:46.014188Z","caller":"mvcc/hash.go:137","msg":"storing new hash","hash":1601956741,"revision":9952,"compact-revision":7574}