level=error ts=2026-03-19T11:15:00.651166088Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.monitoring.svc err="lookup _memcached-client._tcp.loki-results-cache.monitoring.svc on 10.96.0.10:53: no such host"
level=error ts=2026-03-19T11:15:00.65848198Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.monitoring.svc err="lookup _memcached-client._tcp.loki-results-cache.monitoring.svc on 10.96.0.10:53: no such host"
level=error ts=2026-03-19T11:15:00.661202095Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.monitoring.svc err="lookup _memcached-client._tcp.loki-results-cache.monitoring.svc on 10.96.0.10:53: no such host"
level=error ts=2026-03-19T11:15:00.668558848Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.monitoring.svc err="lookup _memcached-client._tcp.loki-results-cache.monitoring.svc on 10.96.0.10:53: no such host"
level=error ts=2026-03-19T11:15:00.673038621Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-results-cache.monitoring.svc err="lookup _memcached-client._tcp.loki-results-cache.monitoring.svc on 10.96.0.10:53: no such host"
level=error ts=2026-03-19T11:15:00.680202348Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.monitoring.svc err="lookup _memcached-client._tcp.loki-chunks-cache.monitoring.svc on 10.96.0.10:53: no such host"
ts=2026-03-19T11:15:00.685528215Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host"
level=warn ts=2026-03-19T11:15:00.685585817Z caller=memberlist_client.go:561 msg="memberlist fast-join finished" joined_nodes=0 elapsed_time=7.9883ms
ts=2026-03-19T11:15:00.693113544Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host"
level=warn ts=2026-03-19T11:15:00.693170786Z caller=memberlist_client.go:607 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host\n\n"
level=error ts=2026-03-19T11:15:00.824564446Z caller=ratestore.go:109 msg="error getting ingester clients" err="empty ring"
level=error ts=2026-03-19T11:15:01.064270609Z caller=compat.go:78 user=fake rule_name=NovaCellNotResponding rule_type=alerting query="(count_over_time({pod_label_component=\"compute\"} |= \"not responding and hence is being omitted from the results\"[1m]) > 0)" query_hash=2160893752 msg="rule evaluation failed" err="empty ring"
level=warn name=NovaCellNotResponding index=0 ts=2026-03-19T11:15:01.064329541Z caller=group.go:483 user=fake file=/var/loki/rules-temp/fake/loki-alerting-rules.yaml group=additional-loki-rules msg="Evaluating rule failed" rule="alert: NovaCellNotResponding\nexpr: (count_over_time({pod_label_component=\"compute\"} |= \"not responding and hence\n  is being omitted from the results\"[1m]) > 0)\nlabels:\n  severity: critical\nannotations:\n  summary: Nova Cell is not responding. It can cause port deletion in CAPI.\n" err="rule evaluation failed: empty ring"
ts=2026-03-19T11:15:01.783107403Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host"
level=warn ts=2026-03-19T11:15:01.783163004Z caller=memberlist_client.go:607 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host\n\n"
ts=2026-03-19T11:15:05.128307689Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host"
level=warn ts=2026-03-19T11:15:05.12834541Z caller=memberlist_client.go:607 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host\n\n"
ts=2026-03-19T11:15:12.495396058Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host"
level=warn ts=2026-03-19T11:15:12.495443469Z caller=memberlist_client.go:607 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host\n\n"
ts=2026-03-19T11:15:28.368654104Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host"
level=warn ts=2026-03-19T11:15:28.368691895Z caller=memberlist_client.go:607 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host\n\n"
ts=2026-03-19T11:15:53.416529932Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host"
level=warn ts=2026-03-19T11:15:53.416575713Z caller=memberlist_client.go:607 phase=startup msg="joining memberlist cluster" attempts=6 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist: lookup loki-memberlist on 10.96.0.10:53: no such host\n\n"
level=error ts=2026-03-19T11:16:04.000051611Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '433403' bytes" org_id=fake
level=error ts=2026-03-19T11:16:04.116015005Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '3285533' bytes" org_id=fake
level=error ts=2026-03-19T11:16:04.153758988Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '1418096' bytes" org_id=fake
level=error ts=2026-03-19T11:16:04.156992816Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '852060' bytes" org_id=fake
level=error ts=2026-03-19T11:16:04.268318914Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '382783' bytes" org_id=fake
level=error ts=2026-03-19T12:12:39.58585787Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '433403' bytes" org_id=fake
level=error ts=2026-03-19T12:13:33.78928179Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '3285533' bytes" org_id=fake
level=error ts=2026-03-19T12:13:42.104981118Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '852060' bytes" org_id=fake
level=error ts=2026-03-19T12:13:44.617854825Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '1418096' bytes" org_id=fake
level=error ts=2026-03-19T12:13:55.57890698Z caller=manager.go:49 component=distributor path=write msg="write operation failed" details="Max entry size '262144' bytes exceeded for stream '{node=\"instance\", service_name=\"unknown_service\", source_type=\"journald\", stream=\"syslog\", unit=\"session-4.scope\"}' while adding an entry with length '382783' bytes" org_id=fake