{"level":"info","ts":"2026-03-04T02:18:44Z","logger":"setup","msg":"started controller"}
{"level":"info","ts":"2026-03-04T02:18:44Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2026-03-04T02:18:44Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-03-04T02:18:44Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":9782","secure":false}
{"level":"info","ts":"2026-03-04T02:18:44Z","logger":"rabbitmq-cluster-operator","msg":"attempting to acquire leader lease openstack/rabbitmq-cluster-operator-leader-election..."}
{"level":"error","ts":"2026-03-04T02:19:22Z","logger":"rabbitmq-cluster-operator","msg":"Failed to update lock: etcdserver: request timed out","stacktrace":"k8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:369\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:252\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/backoff.go:226\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/backoff.go:227\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/backoff.go:204\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:251\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:208\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).startLeaderElection.func3\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/manager/internal.go:598"}
{"level":"info","ts":"2026-03-04T02:19:35Z","logger":"rabbitmq-cluster-operator","msg":"successfully acquired lease openstack/rabbitmq-cluster-operator-leader-election"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting EventSource","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","source":"kind source: *v1beta1.RabbitmqCluster"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting EventSource","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","source":"kind source: *v1.StatefulSet"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting EventSource","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","source":"kind source: *v1.ConfigMap"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting EventSource","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting EventSource","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","source":"kind source: *v1.Role"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting EventSource","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","source":"kind source: *v1.RoleBinding"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting EventSource","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting EventSource","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting Controller","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster"}
{"level":"info","ts":"2026-03-04T02:19:35Z","msg":"Starting workers","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","worker count":1}
{"level":"info","ts":"2026-03-04T02:19:43Z","msg":"Start reconciling","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","RabbitmqCluster":{"name":"rabbitmq-nova","namespace":"openstack"},"namespace":"openstack","name":"rabbitmq-nova","reconcileID":"b4d0f205-fa80-470f-a2d8-b097261e7f66"}
{"level":"info","ts":"2026-03-04T02:19:43Z","msg":"updated resource rabbitmq-nova-nodes of Type *v1.Service","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","RabbitmqCluster":{"name":"rabbitmq-nova","namespace":"openstack"},"namespace":"openstack","name":"rabbitmq-nova","reconcileID":"b4d0f205-fa80-470f-a2d8-b097261e7f66"}
{"level":"info","ts":"2026-03-04T02:19:54Z","msg":"updated resource rabbitmq-nova of Type *v1.Service","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","RabbitmqCluster":{"name":"rabbitmq-nova","namespace":"openstack"},"namespace":"openstack","name":"rabbitmq-nova","reconcileID":"b4d0f205-fa80-470f-a2d8-b097261e7f66"}
{"level":"info","ts":"2026-03-04T02:19:54Z","logger":"statefulset.RabbitmqCluster","msg":"Warning: Memory request and limit are not equal for \"rabbitmq-nova-server\". It is recommended that they be set to the same value"}
{"level":"error","ts":"2026-03-04T02:19:57Z","logger":"rabbitmq-cluster-operator","msg":"error retrieving resource lock openstack/rabbitmq-cluster-operator-leader-election: Get \"https://10.96.0.1:443/apis/coordination.k8s.io/v1/namespaces/openstack/leases/rabbitmq-cluster-operator-leader-election\": context deadline exceeded","stacktrace":"k8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:332\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew.func1.1\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:275\nk8s.io/apimachinery/pkg/util/wait.PollImmediateUntil.ConditionFunc.WithContext.func1\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/wait.go:109\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/wait.go:154\nk8s.io/apimachinery/pkg/util/wait.poll\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/poll.go:245\nk8s.io/apimachinery/pkg/util/wait.PollImmediateUntilWithContext\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/poll.go:200\nk8s.io/apimachinery/pkg/util/wait.PollImmediateUntil\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/poll.go:187\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:274\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/backoff.go:226\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/backoff.go:227\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/backoff.go:204\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/go/pkg/mod/k8s.io/apimachinery@v0.29.3/pkg/util/wait/backoff.go:161\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:271\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\t/go/pkg/mod/k8s.io/client-go@v0.29.2/tools/leaderelection/leaderelection.go:214\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).startLeaderElection.func3\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/manager/internal.go:598"}
{"level":"info","ts":"2026-03-04T02:19:57Z","logger":"rabbitmq-cluster-operator","msg":"failed to renew lease openstack/rabbitmq-cluster-operator-leader-election: timed out waiting for the condition"}
{"level":"info","ts":"2026-03-04T02:19:57Z","msg":"Stopping and waiting for non leader election runnables"}
{"level":"info","ts":"2026-03-04T02:19:57Z","msg":"Stopping and waiting for leader election runnables"}
{"level":"info","ts":"2026-03-04T02:19:57Z","msg":"Stopping and waiting for caches"}
{"level":"info","ts":"2026-03-04T02:19:57Z","msg":"Stopping and waiting for webhooks"}
{"level":"info","ts":"2026-03-04T02:19:57Z","msg":"Stopping and waiting for HTTP servers"}
{"level":"info","ts":"2026-03-04T02:19:57Z","msg":"Wait completed, proceeding to shutdown the manager"}
{"level":"info","ts":"2026-03-04T02:19:57Z","logger":"controller-runtime.metrics","msg":"Shutting down metrics server with timeout of 1 minute"}
{"level":"error","ts":"2026-03-04T02:19:57Z","msg":"Could not read sts","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","RabbitmqCluster":{"name":"rabbitmq-nova","namespace":"openstack"},"namespace":"openstack","name":"rabbitmq-nova","reconcileID":"b4d0f205-fa80-470f-a2d8-b097261e7f66","error":"failed to determine existing STS capactiy: Get \"https://10.96.0.1:443/apis/apps/v1/namespaces/openstack/statefulsets/rabbitmq-nova-server\": context canceled","stacktrace":"github.com/rabbitmq/cluster-operator/v2/internal/scaling.PersistenceScaler.Scale\n\t/workspace/internal/scaling/scaling.go:38\ngithub.com/rabbitmq/cluster-operator/v2/controllers.(*RabbitmqClusterReconciler).reconcilePVC\n\t/workspace/controllers/reconcile_persistence.go:18\ngithub.com/rabbitmq/cluster-operator/v2/controllers.(*RabbitmqClusterReconciler).Reconcile\n\t/workspace/controllers/rabbitmqcluster_controller.go:211\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/internal/controller/controller.go:227"}
{"level":"error","ts":"2026-03-04T02:19:57Z","msg":"Failed to scale PVCs: failed to determine existing STS capactiy: Get \"https://10.96.0.1:443/apis/apps/v1/namespaces/openstack/statefulsets/rabbitmq-nova-server\": context canceled","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster","RabbitmqCluster":{"name":"rabbitmq-nova","namespace":"openstack"},"namespace":"openstack","name":"rabbitmq-nova","reconcileID":"b4d0f205-fa80-470f-a2d8-b097261e7f66","error":"hit an error while scaling PVC capacity: failed to determine existing STS capactiy: Get \"https://10.96.0.1:443/apis/apps/v1/namespaces/openstack/statefulsets/rabbitmq-nova-server\": context canceled","stacktrace":"github.com/rabbitmq/cluster-operator/v2/controllers.(*RabbitmqClusterReconciler).reconcilePVC\n\t/workspace/controllers/reconcile_persistence.go:21\ngithub.com/rabbitmq/cluster-operator/v2/controllers.(*RabbitmqClusterReconciler).Reconcile\n\t/workspace/controllers/rabbitmqcluster_controller.go:211\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.3/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":"2026-03-04T02:19:57Z","msg":"Shutdown signal received, waiting for all workers to finish","controller":"rabbitmqcluster","controllerGroup":"rabbitmq.com","controllerKind":"RabbitmqCluster"}
{"level":"error","ts":"2026-03-04T02:19:57Z","logger":"setup","msg":"problem running manager","error":"leader election lost","stacktrace":"main.main\n\t/workspace/main.go:218\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"}