all: children: cephs: hosts: instance: null computes: hosts: instance: null controllers: hosts: instance: null zuul_unreachable: hosts: {} hosts: instance: ansible_connection: ssh ansible_host: 162.253.55.110 ansible_port: 22 ansible_python_interpreter: auto ansible_user: zuul ceph_conf_overrides: - option: mon allow pool size one section: global value: true - option: osd crush chooseleaf type section: global value: 0 - option: auth allow insecure global id reclaim section: mon value: false ceph_csi_rbd_helm_values: provisioner: replicaCount: 1 ceph_fsid: 4837cbf8-4f90-4300-b3f6-726c9b9f89b4 ceph_osd_devices: - /dev/ceph-{{ inventory_hostname_short }}-osd0/data - /dev/ceph-{{ inventory_hostname_short }}-osd1/data - /dev/ceph-{{ inventory_hostname_short }}-osd2/data cilium_helm_values: operator: replicas: 1 cilium_ipv4_cidr: 172.24.0.0/16 csi_driver: rbd kube_vip_address: 172.17.0.100 kube_vip_interface: '{{ ansible_facts[''default_ipv4''].interface }}' kubernetes_hostname: '{{ ansible_facts[''default_ipv4''].address }}' molecule_scenario: csi nodepool: az: nova cloud: public external_id: ac1dbf35-f6fe-421e-b31b-1348ca275fe4 host_id: 7b8a4cf3090e32af994f4889a62f1003a5f74592d8295d185d35c3b9 interface_ip: 162.253.55.110 label: ubuntu-jammy node_properties: {} private_ipv4: 162.253.55.110 private_ipv6: null provider: yul1 public_ipv4: 162.253.55.110 public_ipv6: 2604:e100:1:0:f816:3eff:fed5:2289 region: ca-ymq-1 slot: null zuul_node: az: nova cloud: public external_id: ac1dbf35-f6fe-421e-b31b-1348ca275fe4 host_id: 7b8a4cf3090e32af994f4889a62f1003a5f74592d8295d185d35c3b9 interface_ip: 162.253.55.110 label: ubuntu-jammy node_properties: {} private_ipv4: 162.253.55.110 private_ipv6: null provider: yul1 public_ipv4: 162.253.55.110 public_ipv6: 2604:e100:1:0:f816:3eff:fed5:2289 region: ca-ymq-1 slot: null uuid: null vars: ceph_conf_overrides: - option: mon allow pool size one section: global value: true - option: osd crush chooseleaf type section: global value: 0 - option: auth allow insecure global id reclaim section: mon value: false ceph_csi_rbd_helm_values: provisioner: replicaCount: 1 ceph_fsid: 4837cbf8-4f90-4300-b3f6-726c9b9f89b4 ceph_osd_devices: - /dev/ceph-{{ inventory_hostname_short }}-osd0/data - /dev/ceph-{{ inventory_hostname_short }}-osd1/data - /dev/ceph-{{ inventory_hostname_short }}-osd2/data cilium_helm_values: operator: replicas: 1 cilium_ipv4_cidr: 172.24.0.0/16 csi_driver: rbd kube_vip_address: 172.17.0.100 kube_vip_interface: '{{ ansible_facts[''default_ipv4''].interface }}' kubernetes_hostname: '{{ ansible_facts[''default_ipv4''].address }}' molecule_scenario: csi zuul: _inheritance_path: - '' - '' - '' - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: d7988bb3149c4e688f8766d7b608adcb build_refs: - branch: main change: '3629' change_message: "fix(monitoring): exclude Donor state from MySQLGaleraOutOfSync alert\n\n## Problem\n\nDuring PXC operator backups, a Galera node enters Donor state (`wsrep_local_state=2`) for State Snapshot Transfer (SST). The upstream `MySQLGaleraOutOfSync` alert fires whenever `wsrep_local_state != 4`, causing false positives during backups since the Donor state is not covered by the `wsrep_desync` check.\n\n## Solution\n\nOverride the upstream `MySQLGaleraOutOfSync` alert expression in-place within the `GaleraAlerts` group to also exclude state 2 (Donor):\n\n```\nmysql_global_status_wsrep_local_state != 4\nAND mysql_global_status_wsrep_local_state != 2\nAND mysql_global_variables_wsrep_desync == 0\n```\n\nState 2 (Donor) is a normal transient state during SST or PXC operator backups. The separate `MySQLGaleraDonorFallingBehind` alert already covers problematic donor scenarios.\n\n## Changes\n\n- **`mixins.libsonnet`**: Restructured the mysqld mixin to patch the upstream `MySQLGaleraOutOfSync` rule in-place rather than disabling and re-adding it, which avoids the `disabledAlerts` filter conflict\n- **`monitoring.rst`**: Added runbook entry with description, root causes, and 5 diagnostic/remediation steps\n- **`tests.yml`**: Added 4 unit tests:\n - Synced state (4) \u2192 no alert\n - Donor state (2) during backup \u2192 no alert (the fix)\n - `wsrep_desync` enabled \u2192 no alert\n - Joining state (1) \u2192 alert fires with correct labels/annotations" change_url: https://github.com/vexxhost/atmosphere/pull/3629 commit_id: 49bd7d491da0db77a0f715479895d2d36ef5a227 patchset: 49bd7d491da0db77a0f715479895d2d36ef5a227 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null buildset: fee79b64f6d7448eabd11a13b208e4e0 buildset_refs: - branch: main change: '3629' change_message: "fix(monitoring): exclude Donor state from MySQLGaleraOutOfSync alert\n\n## Problem\n\nDuring PXC operator backups, a Galera node enters Donor state (`wsrep_local_state=2`) for State Snapshot Transfer (SST). The upstream `MySQLGaleraOutOfSync` alert fires whenever `wsrep_local_state != 4`, causing false positives during backups since the Donor state is not covered by the `wsrep_desync` check.\n\n## Solution\n\nOverride the upstream `MySQLGaleraOutOfSync` alert expression in-place within the `GaleraAlerts` group to also exclude state 2 (Donor):\n\n```\nmysql_global_status_wsrep_local_state != 4\nAND mysql_global_status_wsrep_local_state != 2\nAND mysql_global_variables_wsrep_desync == 0\n```\n\nState 2 (Donor) is a normal transient state during SST or PXC operator backups. The separate `MySQLGaleraDonorFallingBehind` alert already covers problematic donor scenarios.\n\n## Changes\n\n- **`mixins.libsonnet`**: Restructured the mysqld mixin to patch the upstream `MySQLGaleraOutOfSync` rule in-place rather than disabling and re-adding it, which avoids the `disabledAlerts` filter conflict\n- **`monitoring.rst`**: Added runbook entry with description, root causes, and 5 diagnostic/remediation steps\n- **`tests.yml`**: Added 4 unit tests:\n - Synced state (4) \u2192 no alert\n - Donor state (2) during backup \u2192 no alert (the fix)\n - `wsrep_desync` enabled \u2192 no alert\n - Joining state (1) \u2192 alert fires with correct labels/annotations" change_url: https://github.com/vexxhost/atmosphere/pull/3629 commit_id: 49bd7d491da0db77a0f715479895d2d36ef5a227 patchset: 49bd7d491da0db77a0f715479895d2d36ef5a227 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null change: '3629' change_message: "fix(monitoring): exclude Donor state from MySQLGaleraOutOfSync alert\n\n## Problem\n\nDuring PXC operator backups, a Galera node enters Donor state (`wsrep_local_state=2`) for State Snapshot Transfer (SST). The upstream `MySQLGaleraOutOfSync` alert fires whenever `wsrep_local_state != 4`, causing false positives during backups since the Donor state is not covered by the `wsrep_desync` check.\n\n## Solution\n\nOverride the upstream `MySQLGaleraOutOfSync` alert expression in-place within the `GaleraAlerts` group to also exclude state 2 (Donor):\n\n```\nmysql_global_status_wsrep_local_state != 4\nAND mysql_global_status_wsrep_local_state != 2\nAND mysql_global_variables_wsrep_desync == 0\n```\n\nState 2 (Donor) is a normal transient state during SST or PXC operator backups. The separate `MySQLGaleraDonorFallingBehind` alert already covers problematic donor scenarios.\n\n## Changes\n\n- **`mixins.libsonnet`**: Restructured the mysqld mixin to patch the upstream `MySQLGaleraOutOfSync` rule in-place rather than disabling and re-adding it, which avoids the `disabledAlerts` filter conflict\n- **`monitoring.rst`**: Added runbook entry with description, root causes, and 5 diagnostic/remediation steps\n- **`tests.yml`**: Added 4 unit tests:\n - Synced state (4) \u2192 no alert\n - Donor state (2) during backup \u2192 no alert (the fix)\n - `wsrep_desync` enabled \u2192 no alert\n - Joining state (1) \u2192 alert fires with correct labels/annotations" change_url: https://github.com/vexxhost/atmosphere/pull/3629 child_jobs: [] commit_id: 49bd7d491da0db77a0f715479895d2d36ef5a227 event_id: 27c897a0-0c15-11f1-9f20-c7eca91e3a3d executor: hostname: 3a2793d2bd32 inventory_file: /var/lib/zuul/builds/d7988bb3149c4e688f8766d7b608adcb/ansible/inventory.yaml log_root: /var/lib/zuul/builds/d7988bb3149c4e688f8766d7b608adcb/work/logs result_data_file: /var/lib/zuul/builds/d7988bb3149c4e688f8766d7b608adcb/work/results.json src_root: /var/lib/zuul/builds/d7988bb3149c4e688f8766d7b608adcb/work/src work_root: /var/lib/zuul/builds/d7988bb3149c4e688f8766d7b608adcb/work include_vars: [] items: - branch: main change: '3629' change_message: "fix(monitoring): exclude Donor state from MySQLGaleraOutOfSync alert\n\n## Problem\n\nDuring PXC operator backups, a Galera node enters Donor state (`wsrep_local_state=2`) for State Snapshot Transfer (SST). The upstream `MySQLGaleraOutOfSync` alert fires whenever `wsrep_local_state != 4`, causing false positives during backups since the Donor state is not covered by the `wsrep_desync` check.\n\n## Solution\n\nOverride the upstream `MySQLGaleraOutOfSync` alert expression in-place within the `GaleraAlerts` group to also exclude state 2 (Donor):\n\n```\nmysql_global_status_wsrep_local_state != 4\nAND mysql_global_status_wsrep_local_state != 2\nAND mysql_global_variables_wsrep_desync == 0\n```\n\nState 2 (Donor) is a normal transient state during SST or PXC operator backups. The separate `MySQLGaleraDonorFallingBehind` alert already covers problematic donor scenarios.\n\n## Changes\n\n- **`mixins.libsonnet`**: Restructured the mysqld mixin to patch the upstream `MySQLGaleraOutOfSync` rule in-place rather than disabling and re-adding it, which avoids the `disabledAlerts` filter conflict\n- **`monitoring.rst`**: Added runbook entry with description, root causes, and 5 diagnostic/remediation steps\n- **`tests.yml`**: Added 4 unit tests:\n - Synced state (4) \u2192 no alert\n - Donor state (2) during backup \u2192 no alert (the fix)\n - `wsrep_desync` enabled \u2192 no alert\n - Joining state (1) \u2192 alert fires with correct labels/annotations" change_url: https://github.com/vexxhost/atmosphere/pull/3629 commit_id: 49bd7d491da0db77a0f715479895d2d36ef5a227 patchset: 49bd7d491da0db77a0f715479895d2d36ef5a227 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null job: atmosphere-molecule-csi-rbd jobtags: [] max_attempts: 3 message: Zml4KG1vbml0b3JpbmcpOiBleGNsdWRlIERvbm9yIHN0YXRlIGZyb20gTXlTUUxHYWxlcmFPdXRPZlN5bmMgYWxlcnQKCiMjIFByb2JsZW0KCkR1cmluZyBQWEMgb3BlcmF0b3IgYmFja3VwcywgYSBHYWxlcmEgbm9kZSBlbnRlcnMgRG9ub3Igc3RhdGUgKGB3c3JlcF9sb2NhbF9zdGF0ZT0yYCkgZm9yIFN0YXRlIFNuYXBzaG90IFRyYW5zZmVyIChTU1QpLiBUaGUgdXBzdHJlYW0gYE15U1FMR2FsZXJhT3V0T2ZTeW5jYCBhbGVydCBmaXJlcyB3aGVuZXZlciBgd3NyZXBfbG9jYWxfc3RhdGUgIT0gNGAsIGNhdXNpbmcgZmFsc2UgcG9zaXRpdmVzIGR1cmluZyBiYWNrdXBzIHNpbmNlIHRoZSBEb25vciBzdGF0ZSBpcyBub3QgY292ZXJlZCBieSB0aGUgYHdzcmVwX2Rlc3luY2AgY2hlY2suCgojIyBTb2x1dGlvbgoKT3ZlcnJpZGUgdGhlIHVwc3RyZWFtIGBNeVNRTEdhbGVyYU91dE9mU3luY2AgYWxlcnQgZXhwcmVzc2lvbiBpbi1wbGFjZSB3aXRoaW4gdGhlIGBHYWxlcmFBbGVydHNgIGdyb3VwIHRvIGFsc28gZXhjbHVkZSBzdGF0ZSAyIChEb25vcik6CgpgYGAKbXlzcWxfZ2xvYmFsX3N0YXR1c193c3JlcF9sb2NhbF9zdGF0ZSAhPSA0CkFORCBteXNxbF9nbG9iYWxfc3RhdHVzX3dzcmVwX2xvY2FsX3N0YXRlICE9IDIKQU5EIG15c3FsX2dsb2JhbF92YXJpYWJsZXNfd3NyZXBfZGVzeW5jID09IDAKYGBgCgpTdGF0ZSAyIChEb25vcikgaXMgYSBub3JtYWwgdHJhbnNpZW50IHN0YXRlIGR1cmluZyBTU1Qgb3IgUFhDIG9wZXJhdG9yIGJhY2t1cHMuIFRoZSBzZXBhcmF0ZSBgTXlTUUxHYWxlcmFEb25vckZhbGxpbmdCZWhpbmRgIGFsZXJ0IGFscmVhZHkgY292ZXJzIHByb2JsZW1hdGljIGRvbm9yIHNjZW5hcmlvcy4KCiMjIENoYW5nZXMKCi0gKipgbWl4aW5zLmxpYnNvbm5ldGAqKjogUmVzdHJ1Y3R1cmVkIHRoZSBteXNxbGQgbWl4aW4gdG8gcGF0Y2ggdGhlIHVwc3RyZWFtIGBNeVNRTEdhbGVyYU91dE9mU3luY2AgcnVsZSBpbi1wbGFjZSByYXRoZXIgdGhhbiBkaXNhYmxpbmcgYW5kIHJlLWFkZGluZyBpdCwgd2hpY2ggYXZvaWRzIHRoZSBgZGlzYWJsZWRBbGVydHNgIGZpbHRlciBjb25mbGljdAotICoqYG1vbml0b3JpbmcucnN0YCoqOiBBZGRlZCBydW5ib29rIGVudHJ5IHdpdGggZGVzY3JpcHRpb24sIHJvb3QgY2F1c2VzLCBhbmQgNSBkaWFnbm9zdGljL3JlbWVkaWF0aW9uIHN0ZXBzCi0gKipgdGVzdHMueW1sYCoqOiBBZGRlZCA0IHVuaXQgdGVzdHM6CiAgLSBTeW5jZWQgc3RhdGUgKDQpIOKGkiBubyBhbGVydAogIC0gRG9ub3Igc3RhdGUgKDIpIGR1cmluZyBiYWNrdXAg4oaSIG5vIGFsZXJ0ICh0aGUgZml4KQogIC0gYHdzcmVwX2Rlc3luY2AgZW5hYmxlZCDihpIgbm8gYWxlcnQKICAtIEpvaW5pbmcgc3RhdGUgKDEpIOKGkiBhbGVydCBmaXJlcyB3aXRoIGNvcnJlY3QgbGFiZWxzL2Fubm90YXRpb25z patchset: 49bd7d491da0db77a0f715479895d2d36ef5a227 pipeline: check playbook_context: playbook_projects: trusted/project_0/vexxhost.dev/zuul-config: canonical_name: vexxhost.dev/zuul-config checkout: main commit: 9052b5a7781b3346e4cffd452a54448cbff54d8b trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 571c0efa3491d12ecb8fc1169c510716d55c0fc2 trusted/project_2/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: a6e68243e02ef030ce5e75f8b67630880c475f33 untrusted/project_0/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: a6e68243e02ef030ce5e75f8b67630880c475f33 untrusted/project_1/vexxhost.dev/zuul-config: canonical_name: vexxhost.dev/zuul-config checkout: main commit: 9052b5a7781b3346e4cffd452a54448cbff54d8b untrusted/project_2/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 571c0efa3491d12ecb8fc1169c510716d55c0fc2 untrusted/project_3/github.com/vexxhost/atmosphere: canonical_name: github.com/vexxhost/atmosphere checkout: main commit: 49bd7d491da0db77a0f715479895d2d36ef5a227 untrusted/project_4/opendev.org/openstack/openstack-helm: canonical_name: opendev.org/openstack/openstack-helm checkout: master commit: de768d35e689e9f68bef0bf3029725ba685fa28f playbooks: - path: untrusted/project_0/github.com/vexxhost/zuul-jobs/playbooks/molecule/run.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_1/zuul-jobs/roles - checkout: main checkout_description: playbook branch link_name: ansible/playbook_0/role_2/zuul-jobs link_target: untrusted/project_0/github.com/vexxhost/zuul-jobs role_path: ansible/playbook_0/role_2/zuul-jobs/roles post_review: false post_timeout: null pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere projects: github.com/vexxhost/atmosphere: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere checkout: main checkout_description: zuul branch commit: 49bd7d491da0db77a0f715479895d2d36ef5a227 name: vexxhost/atmosphere required: false short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere ref: refs/pull/3629/head resources: {} tenant: oss timeout: 1800 topic: null voting: true