------------------------------------------------------------------------------- NGINX Ingress controller Release: v1.12.1 Build: 51c2b819690bbf1709b844dbf321a9acf6eda5a7 Repository: https://github.com/kubernetes/ingress-nginx nginx version: nginx/1.25.5 ------------------------------------------------------------------------------- W0529 11:26:41.941802 7 client_config.go:667] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work. I0529 11:26:41.941942 7 main.go:205] "Creating API client" host="https://10.96.0.1:443" I0529 11:26:41.946932 7 main.go:248] "Running in Kubernetes cluster" major="1" minor="28" git="v1.28.13" state="clean" commit="024ab2a13ff700705efd3ab63a0e27c38c462070" platform="linux/amd64" I0529 11:26:41.950563 7 main.go:83] "Valid default backend" service="ingress-nginx/ingress-nginx-defaultbackend" I0529 11:26:42.241672 7 main.go:101] "SSL fake certificate created" file="/etc/ingress-controller/ssl/default-fake-certificate.pem" I0529 11:26:42.260637 7 ssl.go:535] "loading tls certificate" path="/usr/local/certificates/cert" key="/usr/local/certificates/key" I0529 11:26:42.275581 7 nginx.go:271] "Starting NGINX Ingress controller" I0529 11:26:42.292651 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-controller", UID:"bf5c1317-abb6-463f-8e42-1d110baf27bd", APIVersion:"v1", ResourceVersion:"1499", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-controller I0529 11:26:42.301524 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-tcp", UID:"0736aa41-1527-41a9-9066-5ff5f6368ce7", APIVersion:"v1", ResourceVersion:"1497", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-tcp I0529 11:26:42.301634 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-udp", UID:"2540b7d9-986e-4f92-b410-adfd00309dfd", APIVersion:"v1", ResourceVersion:"1498", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-udp I0529 11:26:43.478895 7 leaderelection.go:257] attempting to acquire leader lease ingress-nginx/ingress-nginx-leader... I0529 11:26:43.479043 7 nginx.go:317] "Starting NGINX process" I0529 11:26:43.480043 7 nginx.go:337] "Starting validation webhook" address=":7443" certPath="/usr/local/certificates/cert" keyPath="/usr/local/certificates/key" W0529 11:26:43.480570 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:26:43.480607 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0529 11:26:43.480679 7 controller.go:196] "Configuration changes detected, backend reload required" I0529 11:26:43.516837 7 controller.go:216] "Backend successfully reloaded" I0529 11:26:43.516993 7 controller.go:227] "Initial sync, sleeping for 1 second" I0529 11:26:43.517202 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-tdmxc", UID:"8e847382-233f-48e8-a8f9-1fceb4eea055", APIVersion:"v1", ResourceVersion:"1542", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration I0529 11:26:43.700214 7 leaderelection.go:271] successfully acquired lease ingress-nginx/ingress-nginx-leader I0529 11:26:43.700554 7 status.go:85] "New leader elected" identity="ingress-nginx-controller-tdmxc" W0529 11:26:47.294727 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:26:47.294762 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:26:50.631649 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:26:50.631680 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:26:53.962365 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:26:53.962411 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:26:57.297996 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:26:57.298035 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:00.629486 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:00.629522 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:11.333321 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:11.333378 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:14.667426 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:14.667455 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:18.002493 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:18.002581 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:23.299344 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:23.299385 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:26.634380 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:26.634436 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:29.967061 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:29.967099 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:33.299771 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:33.299815 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:36.633751 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:27:36.633786 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store E0529 11:29:03.830218 7 leaderelection.go:429] Failed to update lock optimistically: etcdserver: request timed out, falling back to slow path W0529 11:29:33.974503 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:29:33.974539 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:30:10.080914 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:30:10.082416 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:30:29.813895 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:30:29.813937 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:30:58.308051 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:30:58.308082 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:01.641711 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:01.641762 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:06.805173 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:06.805207 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:10.139063 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:10.139099 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:13.472685 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:13.472733 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:20.725665 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:20.725694 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:24.059662 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:24.059709 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:27.392721 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:27.392786 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:30.725911 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:30.725958 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:38.663323 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:38.663361 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:41.997571 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:41.997613 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:45.330625 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:31:45.330671 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:34:12.384906 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:34:12.384941 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:34:15.363894 7 controller.go:1441] Error getting SSL certificate "auth-system/keycloak.199-19-213-150.nip.io-tls": local SSL certificate auth-system/keycloak.199-19-213-150.nip.io-tls was not found. Using default certificate W0529 11:34:15.363939 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:34:15.363952 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0529 11:34:15.365212 7 admission.go:149] processed ingress via admission controller {testedIngressLength:1 testedIngressTime:0.002s renderingIngressLength:1 renderingIngressTime:0s admissionTime:0.002s testedConfigurationSize:14.7kB} I0529 11:34:15.365236 7 main.go:107] "successfully validated configuration, accepting" ingress="auth-system/keycloak" I0529 11:34:15.370013 7 store.go:440] "Found valid IngressClass" ingress="auth-system/keycloak" ingressclass="atmosphere" W0529 11:34:15.370477 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "auth-system/keycloak.199-19-213-150.nip.io-tls" in local store I0529 11:34:15.370865 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"auth-system", Name:"keycloak", UID:"96897f0d-0213-4f5b-9bef-eef090d247e7", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4146", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0529 11:34:15.718784 7 controller.go:1441] Error getting SSL certificate "auth-system/keycloak.199-19-213-150.nip.io-tls": local SSL certificate auth-system/keycloak.199-19-213-150.nip.io-tls was not found. Using default certificate W0529 11:34:15.718858 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:34:15.718877 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0529 11:34:15.718959 7 controller.go:196] "Configuration changes detected, backend reload required" I0529 11:34:15.772152 7 controller.go:216] "Backend successfully reloaded" I0529 11:34:15.772575 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-tdmxc", UID:"8e847382-233f-48e8-a8f9-1fceb4eea055", APIVersion:"v1", ResourceVersion:"1542", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration I0529 11:34:15.950577 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="auth-system/keycloak.199-19-213-150.nip.io-tls" I0529 11:34:15.951806 7 backend_ssl.go:67] "Adding secret to local store" name="auth-system/keycloak.199-19-213-150.nip.io-tls" W0529 11:34:19.052666 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:34:19.052708 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.19.213.150 - - [29/May/2026:11:34:21 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 2.192 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 2.190 200 586e140d8dadff2b0a539e18c77060cc 199.19.213.150 - - [29/May/2026:11:34:21 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 404 116 "-" "Ansible" 1197 0.173 [auth-system-keycloak-80] [] 10.0.0.59:8080 116 0.174 404 811b75db121a3e0b3dc4d6c4e3746876 199.19.213.150 - - [29/May/2026:11:34:26 +0000] "POST /admin/realms HTTP/1.1" 201 0 "-" "Ansible" 1296 4.708 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 4.708 201 9605b9c0e27f74db3f291b4da355e549 199.19.213.150 - - [29/May/2026:11:34:26 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 200 4246 "-" "Ansible" 1197 0.664 [auth-system-keycloak-80] [] 10.0.0.59:8080 4246 0.665 200 36462ad4c1fcab511cbe71d2288777f7 199.19.213.150 - - [29/May/2026:11:34:28 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.646 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.643 200 2ddd434d827e39be14c3a54b872eb9da 199.19.213.150 - - [29/May/2026:11:34:28 +0000] "GET /admin/realms/atmosphere/client-scopes HTTP/1.1" 200 12500 "-" "Ansible" 1211 0.052 [auth-system-keycloak-80] [] 10.0.0.59:8080 12500 0.051 200 d2de92fa536008ed4274a91220ca4177 199.19.213.150 - - [29/May/2026:11:34:28 +0000] "GET /admin/realms/atmosphere/client-scopes/81b31d69-0e17-4523-a99c-560f2448b313 HTTP/1.1" 200 1278 "-" "Ansible" 1248 0.048 [auth-system-keycloak-80] [] 10.0.0.59:8080 1278 0.048 200 f3abb5b051daaf82c7b439a798678354 199.19.213.150 - - [29/May/2026:11:34:28 +0000] "PUT /admin/realms/atmosphere/client-scopes/81b31d69-0e17-4523-a99c-560f2448b313 HTTP/1.1" 204 0 "-" "Ansible" 1848 0.148 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.146 204 37e14dfabce9e91088ee65a28a6ca166 199.19.213.150 - - [29/May/2026:11:34:28 +0000] "GET /admin/realms/atmosphere/client-scopes/81b31d69-0e17-4523-a99c-560f2448b313/protocol-mappers/models HTTP/1.1" 200 963 "-" "Ansible" 1272 0.077 [auth-system-keycloak-80] [] 10.0.0.59:8080 963 0.077 200 1f5e846a635a31d61f26b9be095893ad 199.19.213.150 - - [29/May/2026:11:34:28 +0000] "GET /admin/realms/atmosphere/client-scopes/81b31d69-0e17-4523-a99c-560f2448b313/protocol-mappers/models/b1cfce86-336d-46d6-bc08-a028e2f2301d HTTP/1.1" 200 366 "-" "Ansible" 1309 0.013 [auth-system-keycloak-80] [] 10.0.0.59:8080 366 0.012 200 b79aa3e695a778c20d36288cdc198d82 199.19.213.150 - - [29/May/2026:11:34:28 +0000] "PUT /admin/realms/atmosphere/client-scopes/81b31d69-0e17-4523-a99c-560f2448b313/protocol-mappers/models/b1cfce86-336d-46d6-bc08-a028e2f2301d HTTP/1.1" 204 0 "-" "Ansible" 1622 0.089 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.088 204 62aac6ceadc8b22c0cf494f0e5ea225d 199.19.213.150 - - [29/May/2026:11:34:28 +0000] "GET /admin/realms/atmosphere/client-scopes/81b31d69-0e17-4523-a99c-560f2448b313 HTTP/1.1" 200 1218 "-" "Ansible" 1248 0.078 [auth-system-keycloak-80] [] 10.0.0.59:8080 1218 0.078 200 a296ae99a01f1efde24ee45e3f183900 I0529 11:34:43.707305 7 status.go:304] "updating Ingress status" namespace="auth-system" ingress="keycloak" currentValue=null newValue=[{"ip":"10.101.118.98"}] I0529 11:34:43.711616 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"auth-system", Name:"keycloak", UID:"96897f0d-0213-4f5b-9bef-eef090d247e7", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4384", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0529 11:34:43.712868 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:34:43.712901 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.19.213.150 - - [29/May/2026:11:34:50 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.499 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.499 200 7679098f987108b9aba139360b42c53a 199.19.213.150 - - [29/May/2026:11:34:50 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 2 "-" "Ansible" 1227 0.016 [auth-system-keycloak-80] [] 10.0.0.59:8080 2 0.015 200 84986488156e9da373060915655da04a 199.19.213.150 - - [29/May/2026:11:34:50 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1657 0.182 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.182 201 fcaa4fe45381fe9982e0ab71dba03862 199.19.213.150 - - [29/May/2026:11:34:50 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.084 [auth-system-keycloak-80] [] 10.0.0.59:8080 1261 0.084 200 6576dd0eb91a98c5e489cb3d83127ed7 199.19.213.150 - - [29/May/2026:11:34:52 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 1.107 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 1.106 200 9767974dcc4aed1572188f59ce281506 199.19.213.150 - - [29/May/2026:11:34:52 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 2 "-" "Ansible" 1222 0.072 [auth-system-keycloak-80] [] 10.0.0.59:8080 2 0.073 200 4d3b6fea1cf00b1bbb2bb221625b8658 199.19.213.150 - - [29/May/2026:11:34:52 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1688 0.112 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.112 201 d7f7bf4bf69673d59ef55cd73dc3285b 199.19.213.150 - - [29/May/2026:11:34:52 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.086 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.085 200 87169d22995c7ee70b7d4b4e4871684d 199.19.213.150 - - [29/May/2026:11:34:53 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.496 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.495 200 1ced3b835fbf9926adb31c59afb778ee 199.19.213.150 - - [29/May/2026:11:34:53 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 2 "-" "Ansible" 1225 0.012 [auth-system-keycloak-80] [] 10.0.0.59:8080 2 0.012 200 b0b4aba3dadc56ed3df2264719917ae9 199.19.213.150 - - [29/May/2026:11:34:53 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1649 0.069 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.069 201 14b13f77191456689b6eb3c4d6b10437 199.19.213.150 - - [29/May/2026:11:34:53 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.078 [auth-system-keycloak-80] [] 10.0.0.59:8080 1251 0.077 200 df4e405e3d0358a11be3cba3a704e644 199.19.213.150 - - [29/May/2026:11:34:54 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.533 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.533 200 5126505e20b618b89455f0ba53529ea3 199.19.213.150 - - [29/May/2026:11:34:54 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.008 [auth-system-keycloak-80] [] 10.0.0.59:8080 1261 0.007 200 29297669bb6f6820a85770e374bf247e 199.19.213.150 - - [29/May/2026:11:34:54 +0000] "GET /admin/realms/atmosphere/clients/a936f781-ef15-457f-a44c-596cc3cc7b92/roles/member HTTP/1.1" 404 119 "-" "Ansible" 1255 0.015 [auth-system-keycloak-80] [] 10.0.0.59:8080 119 0.016 404 f324e6bbc91054ea501b062cbaf87a9a 199.19.213.150 - - [29/May/2026:11:34:54 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.033 [auth-system-keycloak-80] [] 10.0.0.59:8080 1261 0.033 200 2397b62abe46e101f0b77dcb77292406 199.19.213.150 - - [29/May/2026:11:34:54 +0000] "POST /admin/realms/atmosphere/clients/a936f781-ef15-457f-a44c-596cc3cc7b92/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.020 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.020 201 085ea4bac904a4c9ae04b0a62d714893 199.19.213.150 - - [29/May/2026:11:34:54 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.007 [auth-system-keycloak-80] [] 10.0.0.59:8080 1261 0.008 200 e823955bfd137c9600f8871df9f71796 199.19.213.150 - - [29/May/2026:11:34:54 +0000] "GET /admin/realms/atmosphere/clients/a936f781-ef15-457f-a44c-596cc3cc7b92/roles/member HTTP/1.1" 200 166 "-" "Ansible" 1255 0.057 [auth-system-keycloak-80] [] 10.0.0.59:8080 166 0.057 200 ed2c82928726294cd4310f421ca57a90 199.19.213.150 - - [29/May/2026:11:34:55 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.481 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.481 200 d0f24064e730cd3430fa8c34acc1e540 199.19.213.150 - - [29/May/2026:11:34:55 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.028 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.027 200 3dfe205fa47570304872fb2c37bf5182 199.19.213.150 - - [29/May/2026:11:34:55 +0000] "GET /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles/admin HTTP/1.1" 404 119 "-" "Ansible" 1254 0.009 [auth-system-keycloak-80] [] 10.0.0.59:8080 119 0.008 404 c4583f50483eade055fb2bec30c9a34d 199.19.213.150 - - [29/May/2026:11:34:55 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.006 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.006 200 46cee3def62c9d42bd9c30b9b944f639 199.19.213.150 - - [29/May/2026:11:34:55 +0000] "POST /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles HTTP/1.1" 201 0 "-" "Ansible" 1349 0.067 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.067 201 61db627b8d49dd3659c30f8894fb2aae 199.19.213.150 - - [29/May/2026:11:34:55 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.008 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.007 200 6e8287ad80692161550d1553cdaa2aea 199.19.213.150 - - [29/May/2026:11:34:55 +0000] "GET /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles/admin HTTP/1.1" 200 165 "-" "Ansible" 1254 0.015 [auth-system-keycloak-80] [] 10.0.0.59:8080 165 0.015 200 c923be6d8deb51f4749505f99c274d03 199.19.213.150 - - [29/May/2026:11:34:56 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.510 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.510 200 4f3f7fad995c538643649a2a4853e9c8 199.19.213.150 - - [29/May/2026:11:34:56 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.039 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.040 200 65439277e936ac4c094e9baeb3c769cb 199.19.213.150 - - [29/May/2026:11:34:56 +0000] "GET /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles/editor HTTP/1.1" 404 119 "-" "Ansible" 1255 0.010 [auth-system-keycloak-80] [] 10.0.0.59:8080 119 0.010 404 bf6c21530184d9136514d3227fa21bd6 199.19.213.150 - - [29/May/2026:11:34:56 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.007 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.007 200 c2e3f191531a4de776878e941b3bb144 199.19.213.150 - - [29/May/2026:11:34:56 +0000] "POST /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.057 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.056 201 66ee4eb54beacc377dd0768b8f43f5f6 199.19.213.150 - - [29/May/2026:11:34:56 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.007 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.007 200 0b085a6956381f379fe0487dbdbe341c 199.19.213.150 - - [29/May/2026:11:34:56 +0000] "GET /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles/editor HTTP/1.1" 200 166 "-" "Ansible" 1255 0.071 [auth-system-keycloak-80] [] 10.0.0.59:8080 166 0.070 200 ae7f2dc5bf105ce0f18aeedf887ec40b 199.19.213.150 - - [29/May/2026:11:34:57 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.529 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.529 200 2e8e09a02c65f5d2fac30a9862c9e6a9 199.19.213.150 - - [29/May/2026:11:34:57 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.013 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.014 200 ea0803b9b30d46b6dfae022f46bfaec1 199.19.213.150 - - [29/May/2026:11:34:57 +0000] "GET /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles/viewer HTTP/1.1" 404 119 "-" "Ansible" 1255 0.028 [auth-system-keycloak-80] [] 10.0.0.59:8080 119 0.028 404 f2e2d6c657b749728c5f6f6f3e23bded 199.19.213.150 - - [29/May/2026:11:34:57 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.006 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.006 200 1102864d4ed08677d991a476d71cf77c 199.19.213.150 - - [29/May/2026:11:34:57 +0000] "POST /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.017 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.016 201 228160bad2761e0b8f72d8337f743aa7 199.19.213.150 - - [29/May/2026:11:34:57 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.007 [auth-system-keycloak-80] [] 10.0.0.59:8080 1286 0.007 200 94c39cdef06554b06b7b8020df11d758 199.19.213.150 - - [29/May/2026:11:34:57 +0000] "GET /admin/realms/atmosphere/clients/9e26aedd-44bc-4810-a9df-bc5c9b9911b8/roles/viewer HTTP/1.1" 200 166 "-" "Ansible" 1255 0.027 [auth-system-keycloak-80] [] 10.0.0.59:8080 166 0.027 200 2d4dd8a95ce72ce00f9f4c2fc5cf2f80 199.19.213.150 - - [29/May/2026:11:34:58 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.527 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.526 200 00c5509e60067de75d1fab4e8b076372 199.19.213.150 - - [29/May/2026:11:34:58 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.007 [auth-system-keycloak-80] [] 10.0.0.59:8080 1251 0.006 200 430cc571feb012b57d2bc782786dae1d 199.19.213.150 - - [29/May/2026:11:34:58 +0000] "GET /admin/realms/atmosphere/clients/cb5bd9c5-fba8-48a1-93fa-762c73a6b6c3/roles/member HTTP/1.1" 404 119 "-" "Ansible" 1255 0.010 [auth-system-keycloak-80] [] 10.0.0.59:8080 119 0.010 404 803927ee93cc34a59700c28ea5f9e83a 199.19.213.150 - - [29/May/2026:11:34:58 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.006 [auth-system-keycloak-80] [] 10.0.0.59:8080 1251 0.007 200 8db9b5ca6c04b3d18146a4d7b2329002 199.19.213.150 - - [29/May/2026:11:34:58 +0000] "POST /admin/realms/atmosphere/clients/cb5bd9c5-fba8-48a1-93fa-762c73a6b6c3/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.021 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 0.020 201 eb2351cafd861950824aeb8f3d2ed8bb 199.19.213.150 - - [29/May/2026:11:34:58 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.007 [auth-system-keycloak-80] [] 10.0.0.59:8080 1251 0.007 200 6d806fac273f694722087beda396c9d7 199.19.213.150 - - [29/May/2026:11:34:58 +0000] "GET /admin/realms/atmosphere/clients/cb5bd9c5-fba8-48a1-93fa-762c73a6b6c3/roles/member HTTP/1.1" 200 166 "-" "Ansible" 1255 0.013 [auth-system-keycloak-80] [] 10.0.0.59:8080 166 0.013 200 ebf83851c3a88737264b153bdcf041c7 W0529 11:36:11.417457 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:11.417497 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:11.485778 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:11.485882 7 controller.go:1441] Error getting SSL certificate "monitoring/grafana-tls": local SSL certificate monitoring/grafana-tls was not found. Using default certificate W0529 11:36:11.485942 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:11.485964 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:11.486335 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:11.486395 7 controller.go:1441] Error getting SSL certificate "monitoring/prometheus-tls": local SSL certificate monitoring/prometheus-tls was not found. Using default certificate W0529 11:36:11.486471 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:11.486520 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:11.487347 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. I0529 11:36:11.487943 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.001s renderingIngressLength:2 renderingIngressTime:0.001s admissionTime:0.002s testedConfigurationSize:18.2kB} I0529 11:36:11.487963 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-prometheus" W0529 11:36:11.488234 7 controller.go:1441] Error getting SSL certificate "monitoring/alertmanager-tls": local SSL certificate monitoring/alertmanager-tls was not found. Using default certificate W0529 11:36:11.488283 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:11.488298 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0529 11:36:11.490587 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.003s renderingIngressLength:2 renderingIngressTime:0.001s admissionTime:0.004s testedConfigurationSize:18.2kB} I0529 11:36:11.490630 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-alertmanager" I0529 11:36:11.491305 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.006s renderingIngressLength:2 renderingIngressTime:0s admissionTime:0.006s testedConfigurationSize:18.2kB} I0529 11:36:11.491329 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-grafana" I0529 11:36:11.495711 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-prometheus" ingressclass="atmosphere" I0529 11:36:11.496266 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-prometheus", UID:"172283af-bd0e-40b9-b01e-5e76dffad977", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5053", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0529 11:36:11.496432 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/prometheus-tls" in local store I0529 11:36:11.499352 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-alertmanager" ingressclass="atmosphere" I0529 11:36:11.499526 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-alertmanager", UID:"b8913c3e-2ac3-43eb-b349-523bcae13073", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5058", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0529 11:36:11.499798 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/alertmanager-tls" in local store I0529 11:36:11.499831 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-grafana" ingressclass="atmosphere" I0529 11:36:11.502651 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-grafana", UID:"32d04e6f-8e06-4d4e-89d4-a0bd9b19a2c8", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5059", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0529 11:36:11.503121 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/grafana-tls" in local store I0529 11:36:12.105347 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/alertmanager-tls" I0529 11:36:12.106159 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/alertmanager-tls" I0529 11:36:12.208673 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/grafana-tls" I0529 11:36:12.209386 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/grafana-tls" I0529 11:36:12.971050 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/prometheus-tls" I0529 11:36:12.972373 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/prometheus-tls" W0529 11:36:14.752297 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:14.752364 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:14.752397 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:14.752502 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:14.752541 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0529 11:36:14.752623 7 controller.go:196] "Configuration changes detected, backend reload required" I0529 11:36:14.817660 7 controller.go:216] "Backend successfully reloaded" I0529 11:36:14.817901 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-tdmxc", UID:"8e847382-233f-48e8-a8f9-1fceb4eea055", APIVersion:"v1", ResourceVersion:"1542", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration I0529 11:36:15.051735 7 store.go:645] "secret was updated and it is used in ingress annotations. Parsing" secret="monitoring/alertmanager-tls" I0529 11:36:15.089823 7 backend_ssl.go:59] "Updating secret in local store" name="monitoring/alertmanager-tls" W0529 11:36:18.084798 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:18.084954 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:18.085006 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:18.085137 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:18.085164 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:21.418990 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:21.419046 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:21.419074 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:21.419159 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:21.419176 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:24.751798 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:24.751898 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:24.751934 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:24.752047 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:24.752072 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:28.296028 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:28.296070 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:28.296089 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:28.296151 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:28.296159 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:31.631921 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:31.631986 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:31.632061 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:31.632233 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:31.632267 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:34.962991 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:34.963049 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:34.963078 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:34.963166 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:34.963188 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:38.296774 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:38.296823 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:38.296852 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:38.296946 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:38.296966 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:41.629954 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:41.630022 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:41.630071 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:41.630230 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:41.630269 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0529 11:36:44.963769 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-alertmanager" currentValue=null newValue=[{"ip":"10.101.118.98"}] W0529 11:36:44.965387 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:44.965447 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:44.965484 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:44.965592 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:44.965617 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0529 11:36:44.966038 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-prometheus" currentValue=null newValue=[{"ip":"10.101.118.98"}] I0529 11:36:44.970054 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-grafana" currentValue=null newValue=[{"ip":"10.101.118.98"}] I0529 11:36:45.030743 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-prometheus", UID:"172283af-bd0e-40b9-b01e-5e76dffad977", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5678", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0529 11:36:45.032656 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-alertmanager", UID:"b8913c3e-2ac3-43eb-b349-523bcae13073", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5680", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0529 11:36:45.036896 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-grafana", UID:"32d04e6f-8e06-4d4e-89d4-a0bd9b19a2c8", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5681", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0529 11:36:48.299264 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:48.299333 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:48.299381 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:48.299526 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:48.299556 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:51.629618 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:51.629664 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:51.629689 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:51.629771 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:51.629790 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:54.964572 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:54.964635 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:54.964673 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:54.964784 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:54.964803 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:58.297962 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:36:58.298009 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:36:58.298032 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:36:58.298113 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:36:58.298131 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:01.631210 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:37:01.631273 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:37:01.631309 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:37:01.631416 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:01.631437 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:04.963137 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:37:04.963178 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:37:04.963202 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:37:04.963273 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:04.963289 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:08.296227 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:37:08.296273 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0529 11:37:08.296296 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:37:08.296394 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:08.296414 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:12.555417 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:37:12.555471 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0529 11:37:12.555773 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:12.555788 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 10.0.0.158 - - [29/May/2026:11:37:20 +0000] "GET /realms/atmosphere/.well-known/openid-configuration HTTP/1.1" 200 6432 "-" "Go-http-client/1.1" 161 0.031 [auth-system-keycloak-80] [] 10.0.0.59:8080 6432 0.030 200 4bd508eccd0850b26ce5f576ff780fed 10.0.0.177 - - [29/May/2026:11:37:20 +0000] "GET /realms/atmosphere/.well-known/openid-configuration HTTP/1.1" 200 6432 "-" "Go-http-client/1.1" 161 0.023 [auth-system-keycloak-80] [] 10.0.0.59:8080 6432 0.024 200 9fc069b6cad5f2ab57164bd97d6752f5 W0529 11:37:22.211473 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:37:22.211736 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:22.211790 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:25.545940 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:37:25.546167 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:25.546212 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:29.221565 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0529 11:37:29.221674 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:29.221686 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:32.556256 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:32.556333 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:35.899360 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:37:35.899408 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.19.213.150 - - [29/May/2026:11:37:36 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.477 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.475 200 b70bcdd6ed395578d5b05d702b026df1 199.19.213.150 - - [29/May/2026:11:37:36 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 200 4246 "-" "Ansible" 1197 0.005 [auth-system-keycloak-80] [] 10.0.0.59:8080 4246 0.004 200 b5ac0047e1e3acce739c8f483025922b 199.19.213.150 - - [29/May/2026:11:37:38 +0000] "PUT /admin/realms/atmosphere HTTP/1.1" 204 0 "-" "Ansible" 5709 1.762 [auth-system-keycloak-80] [] 10.0.0.59:8080 0 1.761 204 139f741b49b344720c28284f98f587fb 199.19.213.150 - - [29/May/2026:11:37:38 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 200 4384 "-" "Ansible" 1197 0.117 [auth-system-keycloak-80] [] 10.0.0.59:8080 4384 0.118 200 be9b147378cb222ad6551276a33d44ad 199.19.213.150 - - [29/May/2026:11:37:39 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.556 [auth-system-keycloak-80] [] 10.0.0.59:8080 1928 0.554 200 0b6616827786e13a39505e4ba6ea80e7 199.19.213.150 - - [29/May/2026:11:37:39 +0000] "GET /admin/realms/atmosphere/authentication/required-actions HTTP/1.1" 200 1682 "-" "Ansible" 1229 0.011 [auth-system-keycloak-80] [] 10.0.0.59:8080 1682 0.012 200 1f86b0ce3e27809aaa2d0d68d122ebd3 199.19.213.150 - - [29/May/2026:11:37:40 +0000] "GET /realms/atmosphere/.well-known/openid-configuration HTTP/1.1" 200 6432 "-" "ansible-httpget" 181 0.007 [auth-system-keycloak-80] [] 10.0.0.59:8080 6432 0.008 200 6a07b81931394bdbc29836c693930aee E0529 11:38:00.048454 7 leaderelection.go:429] Failed to update lock optimistically: etcdserver: request timed out, falling back to slow path E0529 11:38:08.050014 7 leaderelection.go:436] error retrieving resource lock ingress-nginx/ingress-nginx-leader: Get "https://10.96.0.1:443/apis/coordination.k8s.io/v1/namespaces/ingress-nginx/leases/ingress-nginx-leader": context deadline exceeded I0529 11:38:08.050060 7 leaderelection.go:297] failed to renew lease ingress-nginx/ingress-nginx-leader: context deadline exceeded E0529 11:38:08.050218 7 status.go:104] "error running poll" err="timed out waiting for the condition" I0529 11:38:08.050244 7 leaderelection.go:257] attempting to acquire leader lease ingress-nginx/ingress-nginx-leader... E0529 11:38:08.097112 7 leaderelection.go:429] Failed to update lock optimistically: Operation cannot be fulfilled on leases.coordination.k8s.io "ingress-nginx-leader": the object has been modified; please apply your changes to the latest version and try again, falling back to slow path I0529 11:38:09.848608 7 leaderelection.go:271] successfully acquired lease ingress-nginx/ingress-nginx-leader W0529 11:38:48.211684 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:38:48.211736 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:38:56.625596 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:38:56.626842 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:39:21.669547 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0529 11:39:21.669587 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store