all: children: cephs: hosts: instance: null computes: hosts: instance: null controllers: hosts: instance: null zuul_unreachable: hosts: {} hosts: instance: ansible_connection: ssh ansible_host: 199.204.45.248 ansible_port: 22 ansible_python_interpreter: auto ansible_user: zuul ceph_conf_overrides: - option: mon allow pool size one section: global value: true - option: osd crush chooseleaf type section: global value: 0 - option: auth allow insecure global id reclaim section: mon value: false ceph_csi_rbd_helm_values: provisioner: replicaCount: 1 ceph_fsid: 4837cbf8-4f90-4300-b3f6-726c9b9f89b4 ceph_osd_devices: - /dev/ceph-{{ inventory_hostname_short }}-osd0/data - /dev/ceph-{{ inventory_hostname_short }}-osd1/data - /dev/ceph-{{ inventory_hostname_short }}-osd2/data cilium_helm_values: operator: replicas: 1 cilium_ipv4_cidr: 172.24.0.0/16 csi_driver: local-path-provisioner kube_vip_address: 172.17.0.100 kube_vip_interface: '{{ ansible_facts[''default_ipv4''].interface }}' kubernetes_hostname: '{{ ansible_facts[''default_ipv4''].address }}' molecule_scenario: csi nodepool: az: nova cloud: public external_id: cb3d4544-b96f-49b4-bb0e-7c567e3b45f6 host_id: be92abac58cdf319c41ec3044bb265315879370c10aa7110cfdbfb5f interface_ip: 199.204.45.248 label: ubuntu-jammy node_properties: {} private_ipv4: 199.204.45.248 private_ipv6: null provider: yul1 public_ipv4: 199.204.45.248 public_ipv6: 2604:e100:1:0:f816:3eff:fe57:e5d8 region: ca-ymq-1 slot: null zuul_node: az: nova cloud: public external_id: cb3d4544-b96f-49b4-bb0e-7c567e3b45f6 host_id: be92abac58cdf319c41ec3044bb265315879370c10aa7110cfdbfb5f interface_ip: 199.204.45.248 label: ubuntu-jammy node_properties: {} private_ipv4: 199.204.45.248 private_ipv6: null provider: yul1 public_ipv4: 199.204.45.248 public_ipv6: 2604:e100:1:0:f816:3eff:fe57:e5d8 region: ca-ymq-1 slot: null uuid: null vars: ceph_conf_overrides: - option: mon allow pool size one section: global value: true - option: osd crush chooseleaf type section: global value: 0 - option: auth allow insecure global id reclaim section: mon value: false ceph_csi_rbd_helm_values: provisioner: replicaCount: 1 ceph_fsid: 4837cbf8-4f90-4300-b3f6-726c9b9f89b4 ceph_osd_devices: - /dev/ceph-{{ inventory_hostname_short }}-osd0/data - /dev/ceph-{{ inventory_hostname_short }}-osd1/data - /dev/ceph-{{ inventory_hostname_short }}-osd2/data cilium_helm_values: operator: replicas: 1 cilium_ipv4_cidr: 172.24.0.0/16 csi_driver: local-path-provisioner kube_vip_address: 172.17.0.100 kube_vip_interface: '{{ ansible_facts[''default_ipv4''].interface }}' kubernetes_hostname: '{{ ansible_facts[''default_ipv4''].address }}' molecule_scenario: csi zuul: _inheritance_path: - '' - '' - '' - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: 279094e91c794ab4b434e84a4ade2776 build_refs: - branch: main change: '3640' change_message: "feat(monitoring): replace CoreDNS mixin with burn-rate SLO alerts\n\n## Summary\n\nReplaces the upstream CoreDNS mixin alerts with custom burn-rate SLO alerts following the project's alerting philosophy.\n\n### Problem\n\nThe upstream CoreDNS mixin produced **cascading alert storms** during upstream DNS blips. A single upstream DNS hiccup triggered up to 5 alerts simultaneously (`CoreDNSForwardHealthcheckFailureCount`, `CoreDNSForwardHealthcheckBrokenCount`, `CoreDNSForwardErrorsHigh`, `CoreDNSErrorsHigh`, `CoreDNSLatencyHigh`) \u2014 all with 100% flap rates in production (ext-nurucloud). The alerts were:\n\n- **Cause-based** (healthcheck counters) instead of symptom-based\n- Using **static thresholds** instead of burn-rate/SLO detection\n- Missing **documentation**, **tests**, and **runbook URLs**\n- Mapped to incorrect severities (e.g., forward errors to one upstream as P1)\n\n### Solution\n\nNew custom `coredns.libsonnet` with burn-rate SLO alerts matching the NGINX Ingress pattern:\n\n| Alert | Severity | Burn Rate | Long Window | Short Window |\n|-------|----------|-----------|-------------|--------------|\n| `CoreDNSCriticalErrorBudgetBurn` | P1 | 14.4x | 1h | 5m |\n| `CoreDNSHighErrorBudgetBurn` | P2 | 6x | 6h | 30m |\n| `CoreDNSModerateErrorBudgetBurn` | P3 | 3x | 1d | 2h |\n| `CoreDNSLowErrorBudgetBurn` | P4 | 1x | 3d | 6h |\n| `CoreDNSDown` | P3 | N/A | N/A | N/A |\n\nAlertmanager inhibition rules ensure higher-tier burn-rate alerts suppress lower tiers.\n\n### Checklist\n\n- [x] All alerts have documentation entries in `monitoring.rst`\n- [x] All entries include description, root causes, and remediation steps\n- [x] P1-P3 alerts have `runbook_url` annotations\n- [x] Summary and description annotations follow the format guidelines\n- [x] All alerts have negative and positive test cases\n- [x] Test expectations include all annotations\n- [x] Vale linting passes with no new errors\n- [x] All CoreDNS tests pass" change_url: https://github.com/vexxhost/atmosphere/pull/3640 commit_id: da4ef011f2cc35f630525a82dfa06d331f3bf690 patchset: da4ef011f2cc35f630525a82dfa06d331f3bf690 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null buildset: 0fa6f29db850412b99735bb4dd6e635d buildset_refs: - branch: main change: '3640' change_message: "feat(monitoring): replace CoreDNS mixin with burn-rate SLO alerts\n\n## Summary\n\nReplaces the upstream CoreDNS mixin alerts with custom burn-rate SLO alerts following the project's alerting philosophy.\n\n### Problem\n\nThe upstream CoreDNS mixin produced **cascading alert storms** during upstream DNS blips. A single upstream DNS hiccup triggered up to 5 alerts simultaneously (`CoreDNSForwardHealthcheckFailureCount`, `CoreDNSForwardHealthcheckBrokenCount`, `CoreDNSForwardErrorsHigh`, `CoreDNSErrorsHigh`, `CoreDNSLatencyHigh`) \u2014 all with 100% flap rates in production (ext-nurucloud). The alerts were:\n\n- **Cause-based** (healthcheck counters) instead of symptom-based\n- Using **static thresholds** instead of burn-rate/SLO detection\n- Missing **documentation**, **tests**, and **runbook URLs**\n- Mapped to incorrect severities (e.g., forward errors to one upstream as P1)\n\n### Solution\n\nNew custom `coredns.libsonnet` with burn-rate SLO alerts matching the NGINX Ingress pattern:\n\n| Alert | Severity | Burn Rate | Long Window | Short Window |\n|-------|----------|-----------|-------------|--------------|\n| `CoreDNSCriticalErrorBudgetBurn` | P1 | 14.4x | 1h | 5m |\n| `CoreDNSHighErrorBudgetBurn` | P2 | 6x | 6h | 30m |\n| `CoreDNSModerateErrorBudgetBurn` | P3 | 3x | 1d | 2h |\n| `CoreDNSLowErrorBudgetBurn` | P4 | 1x | 3d | 6h |\n| `CoreDNSDown` | P3 | N/A | N/A | N/A |\n\nAlertmanager inhibition rules ensure higher-tier burn-rate alerts suppress lower tiers.\n\n### Checklist\n\n- [x] All alerts have documentation entries in `monitoring.rst`\n- [x] All entries include description, root causes, and remediation steps\n- [x] P1-P3 alerts have `runbook_url` annotations\n- [x] Summary and description annotations follow the format guidelines\n- [x] All alerts have negative and positive test cases\n- [x] Test expectations include all annotations\n- [x] Vale linting passes with no new errors\n- [x] All CoreDNS tests pass" change_url: https://github.com/vexxhost/atmosphere/pull/3640 commit_id: da4ef011f2cc35f630525a82dfa06d331f3bf690 patchset: da4ef011f2cc35f630525a82dfa06d331f3bf690 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null change: '3640' change_message: "feat(monitoring): replace CoreDNS mixin with burn-rate SLO alerts\n\n## Summary\n\nReplaces the upstream CoreDNS mixin alerts with custom burn-rate SLO alerts following the project's alerting philosophy.\n\n### Problem\n\nThe upstream CoreDNS mixin produced **cascading alert storms** during upstream DNS blips. A single upstream DNS hiccup triggered up to 5 alerts simultaneously (`CoreDNSForwardHealthcheckFailureCount`, `CoreDNSForwardHealthcheckBrokenCount`, `CoreDNSForwardErrorsHigh`, `CoreDNSErrorsHigh`, `CoreDNSLatencyHigh`) \u2014 all with 100% flap rates in production (ext-nurucloud). The alerts were:\n\n- **Cause-based** (healthcheck counters) instead of symptom-based\n- Using **static thresholds** instead of burn-rate/SLO detection\n- Missing **documentation**, **tests**, and **runbook URLs**\n- Mapped to incorrect severities (e.g., forward errors to one upstream as P1)\n\n### Solution\n\nNew custom `coredns.libsonnet` with burn-rate SLO alerts matching the NGINX Ingress pattern:\n\n| Alert | Severity | Burn Rate | Long Window | Short Window |\n|-------|----------|-----------|-------------|--------------|\n| `CoreDNSCriticalErrorBudgetBurn` | P1 | 14.4x | 1h | 5m |\n| `CoreDNSHighErrorBudgetBurn` | P2 | 6x | 6h | 30m |\n| `CoreDNSModerateErrorBudgetBurn` | P3 | 3x | 1d | 2h |\n| `CoreDNSLowErrorBudgetBurn` | P4 | 1x | 3d | 6h |\n| `CoreDNSDown` | P3 | N/A | N/A | N/A |\n\nAlertmanager inhibition rules ensure higher-tier burn-rate alerts suppress lower tiers.\n\n### Checklist\n\n- [x] All alerts have documentation entries in `monitoring.rst`\n- [x] All entries include description, root causes, and remediation steps\n- [x] P1-P3 alerts have `runbook_url` annotations\n- [x] Summary and description annotations follow the format guidelines\n- [x] All alerts have negative and positive test cases\n- [x] Test expectations include all annotations\n- [x] Vale linting passes with no new errors\n- [x] All CoreDNS tests pass" change_url: https://github.com/vexxhost/atmosphere/pull/3640 child_jobs: [] commit_id: da4ef011f2cc35f630525a82dfa06d331f3bf690 event_id: a9cc8600-0c31-11f1-92d9-da03c05971ff executor: hostname: 3a2793d2bd32 inventory_file: /var/lib/zuul/builds/279094e91c794ab4b434e84a4ade2776/ansible/inventory.yaml log_root: /var/lib/zuul/builds/279094e91c794ab4b434e84a4ade2776/work/logs result_data_file: /var/lib/zuul/builds/279094e91c794ab4b434e84a4ade2776/work/results.json src_root: /var/lib/zuul/builds/279094e91c794ab4b434e84a4ade2776/work/src work_root: /var/lib/zuul/builds/279094e91c794ab4b434e84a4ade2776/work include_vars: [] items: - branch: main change: '3640' change_message: "feat(monitoring): replace CoreDNS mixin with burn-rate SLO alerts\n\n## Summary\n\nReplaces the upstream CoreDNS mixin alerts with custom burn-rate SLO alerts following the project's alerting philosophy.\n\n### Problem\n\nThe upstream CoreDNS mixin produced **cascading alert storms** during upstream DNS blips. A single upstream DNS hiccup triggered up to 5 alerts simultaneously (`CoreDNSForwardHealthcheckFailureCount`, `CoreDNSForwardHealthcheckBrokenCount`, `CoreDNSForwardErrorsHigh`, `CoreDNSErrorsHigh`, `CoreDNSLatencyHigh`) \u2014 all with 100% flap rates in production (ext-nurucloud). The alerts were:\n\n- **Cause-based** (healthcheck counters) instead of symptom-based\n- Using **static thresholds** instead of burn-rate/SLO detection\n- Missing **documentation**, **tests**, and **runbook URLs**\n- Mapped to incorrect severities (e.g., forward errors to one upstream as P1)\n\n### Solution\n\nNew custom `coredns.libsonnet` with burn-rate SLO alerts matching the NGINX Ingress pattern:\n\n| Alert | Severity | Burn Rate | Long Window | Short Window |\n|-------|----------|-----------|-------------|--------------|\n| `CoreDNSCriticalErrorBudgetBurn` | P1 | 14.4x | 1h | 5m |\n| `CoreDNSHighErrorBudgetBurn` | P2 | 6x | 6h | 30m |\n| `CoreDNSModerateErrorBudgetBurn` | P3 | 3x | 1d | 2h |\n| `CoreDNSLowErrorBudgetBurn` | P4 | 1x | 3d | 6h |\n| `CoreDNSDown` | P3 | N/A | N/A | N/A |\n\nAlertmanager inhibition rules ensure higher-tier burn-rate alerts suppress lower tiers.\n\n### Checklist\n\n- [x] All alerts have documentation entries in `monitoring.rst`\n- [x] All entries include description, root causes, and remediation steps\n- [x] P1-P3 alerts have `runbook_url` annotations\n- [x] Summary and description annotations follow the format guidelines\n- [x] All alerts have negative and positive test cases\n- [x] Test expectations include all annotations\n- [x] Vale linting passes with no new errors\n- [x] All CoreDNS tests pass" change_url: https://github.com/vexxhost/atmosphere/pull/3640 commit_id: da4ef011f2cc35f630525a82dfa06d331f3bf690 patchset: da4ef011f2cc35f630525a82dfa06d331f3bf690 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null job: atmosphere-molecule-csi-local-path-provisioner jobtags: [] max_attempts: 3 message: ZmVhdChtb25pdG9yaW5nKTogcmVwbGFjZSBDb3JlRE5TIG1peGluIHdpdGggYnVybi1yYXRlIFNMTyBhbGVydHMKCiMjIFN1bW1hcnkKClJlcGxhY2VzIHRoZSB1cHN0cmVhbSBDb3JlRE5TIG1peGluIGFsZXJ0cyB3aXRoIGN1c3RvbSBidXJuLXJhdGUgU0xPIGFsZXJ0cyBmb2xsb3dpbmcgdGhlIHByb2plY3QncyBhbGVydGluZyBwaGlsb3NvcGh5LgoKIyMjIFByb2JsZW0KClRoZSB1cHN0cmVhbSBDb3JlRE5TIG1peGluIHByb2R1Y2VkICoqY2FzY2FkaW5nIGFsZXJ0IHN0b3JtcyoqIGR1cmluZyB1cHN0cmVhbSBETlMgYmxpcHMuIEEgc2luZ2xlIHVwc3RyZWFtIEROUyBoaWNjdXAgdHJpZ2dlcmVkIHVwIHRvIDUgYWxlcnRzIHNpbXVsdGFuZW91c2x5IChgQ29yZUROU0ZvcndhcmRIZWFsdGhjaGVja0ZhaWx1cmVDb3VudGAsIGBDb3JlRE5TRm9yd2FyZEhlYWx0aGNoZWNrQnJva2VuQ291bnRgLCBgQ29yZUROU0ZvcndhcmRFcnJvcnNIaWdoYCwgYENvcmVETlNFcnJvcnNIaWdoYCwgYENvcmVETlNMYXRlbmN5SGlnaGApIOKAlCBhbGwgd2l0aCAxMDAlIGZsYXAgcmF0ZXMgaW4gcHJvZHVjdGlvbiAoZXh0LW51cnVjbG91ZCkuIFRoZSBhbGVydHMgd2VyZToKCi0gKipDYXVzZS1iYXNlZCoqIChoZWFsdGhjaGVjayBjb3VudGVycykgaW5zdGVhZCBvZiBzeW1wdG9tLWJhc2VkCi0gVXNpbmcgKipzdGF0aWMgdGhyZXNob2xkcyoqIGluc3RlYWQgb2YgYnVybi1yYXRlL1NMTyBkZXRlY3Rpb24KLSBNaXNzaW5nICoqZG9jdW1lbnRhdGlvbioqLCAqKnRlc3RzKiosIGFuZCAqKnJ1bmJvb2sgVVJMcyoqCi0gTWFwcGVkIHRvIGluY29ycmVjdCBzZXZlcml0aWVzIChlLmcuLCBmb3J3YXJkIGVycm9ycyB0byBvbmUgdXBzdHJlYW0gYXMgUDEpCgojIyMgU29sdXRpb24KCk5ldyBjdXN0b20gYGNvcmVkbnMubGlic29ubmV0YCB3aXRoIGJ1cm4tcmF0ZSBTTE8gYWxlcnRzIG1hdGNoaW5nIHRoZSBOR0lOWCBJbmdyZXNzIHBhdHRlcm46Cgp8IEFsZXJ0IHwgU2V2ZXJpdHkgfCBCdXJuIFJhdGUgfCBMb25nIFdpbmRvdyB8IFNob3J0IFdpbmRvdyB8CnwtLS0tLS0tfC0tLS0tLS0tLS18LS0tLS0tLS0tLS18LS0tLS0tLS0tLS0tLXwtLS0tLS0tLS0tLS0tLXwKfCBgQ29yZUROU0NyaXRpY2FsRXJyb3JCdWRnZXRCdXJuYCB8IFAxIHwgMTQuNHggfCAxaCB8IDVtIHwKfCBgQ29yZUROU0hpZ2hFcnJvckJ1ZGdldEJ1cm5gIHwgUDIgfCA2eCB8IDZoIHwgMzBtIHwKfCBgQ29yZUROU01vZGVyYXRlRXJyb3JCdWRnZXRCdXJuYCB8IFAzIHwgM3ggfCAxZCB8IDJoIHwKfCBgQ29yZUROU0xvd0Vycm9yQnVkZ2V0QnVybmAgfCBQNCB8IDF4IHwgM2QgfCA2aCB8CnwgYENvcmVETlNEb3duYCB8IFAzIHwgTi9BIHwgTi9BIHwgTi9BIHwKCkFsZXJ0bWFuYWdlciBpbmhpYml0aW9uIHJ1bGVzIGVuc3VyZSBoaWdoZXItdGllciBidXJuLXJhdGUgYWxlcnRzIHN1cHByZXNzIGxvd2VyIHRpZXJzLgoKIyMjIENoZWNrbGlzdAoKLSBbeF0gQWxsIGFsZXJ0cyBoYXZlIGRvY3VtZW50YXRpb24gZW50cmllcyBpbiBgbW9uaXRvcmluZy5yc3RgCi0gW3hdIEFsbCBlbnRyaWVzIGluY2x1ZGUgZGVzY3JpcHRpb24sIHJvb3QgY2F1c2VzLCBhbmQgcmVtZWRpYXRpb24gc3RlcHMKLSBbeF0gUDEtUDMgYWxlcnRzIGhhdmUgYHJ1bmJvb2tfdXJsYCBhbm5vdGF0aW9ucwotIFt4XSBTdW1tYXJ5IGFuZCBkZXNjcmlwdGlvbiBhbm5vdGF0aW9ucyBmb2xsb3cgdGhlIGZvcm1hdCBndWlkZWxpbmVzCi0gW3hdIEFsbCBhbGVydHMgaGF2ZSBuZWdhdGl2ZSBhbmQgcG9zaXRpdmUgdGVzdCBjYXNlcwotIFt4XSBUZXN0IGV4cGVjdGF0aW9ucyBpbmNsdWRlIGFsbCBhbm5vdGF0aW9ucwotIFt4XSBWYWxlIGxpbnRpbmcgcGFzc2VzIHdpdGggbm8gbmV3IGVycm9ycwotIFt4XSBBbGwgQ29yZUROUyB0ZXN0cyBwYXNz patchset: da4ef011f2cc35f630525a82dfa06d331f3bf690 pipeline: check playbook_context: playbook_projects: trusted/project_0/vexxhost.dev/zuul-config: canonical_name: vexxhost.dev/zuul-config checkout: main commit: 9052b5a7781b3346e4cffd452a54448cbff54d8b trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 571c0efa3491d12ecb8fc1169c510716d55c0fc2 trusted/project_2/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: a6e68243e02ef030ce5e75f8b67630880c475f33 untrusted/project_0/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: a6e68243e02ef030ce5e75f8b67630880c475f33 untrusted/project_1/vexxhost.dev/zuul-config: canonical_name: vexxhost.dev/zuul-config checkout: main commit: 9052b5a7781b3346e4cffd452a54448cbff54d8b untrusted/project_2/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 571c0efa3491d12ecb8fc1169c510716d55c0fc2 untrusted/project_3/github.com/vexxhost/atmosphere: canonical_name: github.com/vexxhost/atmosphere checkout: main commit: da4ef011f2cc35f630525a82dfa06d331f3bf690 untrusted/project_4/opendev.org/openstack/openstack-helm: canonical_name: opendev.org/openstack/openstack-helm checkout: master commit: de768d35e689e9f68bef0bf3029725ba685fa28f playbooks: - path: untrusted/project_0/github.com/vexxhost/zuul-jobs/playbooks/molecule/run.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_1/zuul-jobs/roles - checkout: main checkout_description: playbook branch link_name: ansible/playbook_0/role_2/zuul-jobs link_target: untrusted/project_0/github.com/vexxhost/zuul-jobs role_path: ansible/playbook_0/role_2/zuul-jobs/roles post_review: false post_timeout: null pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere projects: github.com/vexxhost/atmosphere: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere checkout: main checkout_description: zuul branch commit: da4ef011f2cc35f630525a82dfa06d331f3bf690 name: vexxhost/atmosphere required: false short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere ref: refs/pull/3640/head resources: {} tenant: oss timeout: 1800 topic: null voting: true