all: children: cephs: hosts: instance: null computes: hosts: instance: null controllers: hosts: instance: null zuul_unreachable: hosts: {} hosts: instance: ansible_connection: ssh ansible_host: 199.204.45.53 ansible_port: 22 ansible_python_interpreter: auto ansible_user: zuul atmosphere_image_prefix: harbor.atmosphere.dev/ atmosphere_network_backend: openvswitch barbican_helm_values: pod: replicas: api: 1 ceph_conf_overrides: - option: mon allow pool size one section: global value: true - option: osd crush chooseleaf type section: global value: 0 - option: auth allow insecure global id reclaim section: mon value: false ceph_csi_rbd_helm_values: provisioner: replicaCount: 1 ceph_fsid: 4837cbf8-4f90-4300-b3f6-726c9b9f89b4 ceph_osd_devices: - /dev/ceph-{{ inventory_hostname_short }}-osd0/data - /dev/ceph-{{ inventory_hostname_short }}-osd1/data - /dev/ceph-{{ inventory_hostname_short }}-osd2/data ceph_version: 18.2.7 cilium_helm_values: operator: replicas: 1 cinder_helm_values: conf: ceph: pools: backup: replication: 1 cinder.volumes: replication: 1 cinder: DEFAULT: osapi_volume_workers: 2 pod: replicas: api: 1 scheduler: 1 cluster_issuer_type: self-signed coredns_helm_values: replicaCount: 1 csi_driver: local-path-provisioner glance_helm_values: conf: glance: DEFAULT: workers: 2 glance_store: rbd_store_replication: 1 pod: replicas: api: 1 glance_images: - container_format: bare disk_format: raw is_public: true min_disk: 1 name: cirros url: http://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img heat_helm_values: conf: heat: DEFAULT: num_engine_workers: 2 heat_api: workers: 2 heat_api_cfn: workers: 2 heat_api_cloudwatch: workers: 2 pod: replicas: api: 1 cfn: 1 cloudwatch: 1 engine: 1 horizon_helm_values: pod: replicas: server: 1 ingress_nginx_helm_values: controller: config: worker-processes: 2 keystone_helm_values: pod: replicas: api: 1 kube_vip_address: 172.17.0.100 kube_vip_interface: '{{ ansible_facts[''default_ipv4''].interface }}' kubernetes_hostname: '{{ ansible_facts[''default_ipv4''].address }}' kubernetes_keepalived_interface: br-mgmt magnum_helm_values: conf: magnum: api: workers: 2 conductor: workers: 2 pod: replicas: api: 1 conductor: 1 magnum_image_disk_format: qcow2 magnum_images: '[ {{ _magnum_images[-1] }} ]' manila_helm_values: conf: manila: DEFAULT: osapi_share_workers: 2 pod: replicas: api: 1 scheduler: 1 molecule_scenario: aio neutron_helm_values: conf: neutron: DEFAULT: api_workers: 2 metadata_workers: 2 rpc_workers: 2 pod: replicas: rpc_server: 1 server: 1 nodepool: az: nova cloud: public external_id: 76066640-16d9-4955-9e5d-c57d171dd5b1 host_id: be92abac58cdf319c41ec3044bb265315879370c10aa7110cfdbfb5f interface_ip: 199.204.45.53 label: ubuntu-jammy-16 node_properties: {} private_ipv4: 199.204.45.53 private_ipv6: null provider: yul1 public_ipv4: 199.204.45.53 public_ipv6: 2604:e100:1:0:f816:3eff:fe5f:9fe7 region: ca-ymq-1 slot: null nova_helm_values: conf: nova: DEFAULT: metadata_workers: 2 osapi_compute_workers: 2 conductor: workers: 2 scheduler: workers: 2 pod: replicas: api_metadata: 1 conductor: 1 novncproxy: 1 osapi: 1 scheduler: 1 spiceproxy: 1 octavia_helm_values: conf: octavia: controller_worker: workers: 2 octavia_api_uwsgi: uwsgi: processes: 2 pod: replicas: api: 1 housekeeping: 1 worker: 1 ovn_helm_values: conf: auto_bridge_add: br-ex: null pod: replicas: ovn_northd: 1 ovn_ovsdb_nb: 1 ovn_ovsdb_sb: 1 percona_xtradb_cluster_spec: allowUnsafeConfigurations: true haproxy: size: 1 pxc: size: 1 placement_helm_values: conf: placement_api_uwsgi: uwsgi: processes: 2 pod: replicas: api: 1 rook_ceph_cluster_radosgw_spec: dataPool: failureDomain: osd gateway: instances: 1 metadataPool: failureDomain: osd staffeln_helm_values: pod: replicas: api: 1 conductor: 1 valkey_helm_values: replica: replicaCount: 1 zuul_node: az: nova cloud: public external_id: 76066640-16d9-4955-9e5d-c57d171dd5b1 host_id: be92abac58cdf319c41ec3044bb265315879370c10aa7110cfdbfb5f interface_ip: 199.204.45.53 label: ubuntu-jammy-16 node_properties: {} private_ipv4: 199.204.45.53 private_ipv6: null provider: yul1 public_ipv4: 199.204.45.53 public_ipv6: 2604:e100:1:0:f816:3eff:fe5f:9fe7 region: ca-ymq-1 slot: null uuid: null vars: atmosphere_image_prefix: harbor.atmosphere.dev/ atmosphere_network_backend: openvswitch barbican_helm_values: pod: replicas: api: 1 ceph_conf_overrides: - option: mon allow pool size one section: global value: true - option: osd crush chooseleaf type section: global value: 0 - option: auth allow insecure global id reclaim section: mon value: false ceph_csi_rbd_helm_values: provisioner: replicaCount: 1 ceph_fsid: 4837cbf8-4f90-4300-b3f6-726c9b9f89b4 ceph_osd_devices: - /dev/ceph-{{ inventory_hostname_short }}-osd0/data - /dev/ceph-{{ inventory_hostname_short }}-osd1/data - /dev/ceph-{{ inventory_hostname_short }}-osd2/data ceph_version: 18.2.7 cilium_helm_values: operator: replicas: 1 cinder_helm_values: conf: ceph: pools: backup: replication: 1 cinder.volumes: replication: 1 cinder: DEFAULT: osapi_volume_workers: 2 pod: replicas: api: 1 scheduler: 1 cluster_issuer_type: self-signed coredns_helm_values: replicaCount: 1 csi_driver: local-path-provisioner glance_helm_values: conf: glance: DEFAULT: workers: 2 glance_store: rbd_store_replication: 1 pod: replicas: api: 1 glance_images: - container_format: bare disk_format: raw is_public: true min_disk: 1 name: cirros url: http://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img heat_helm_values: conf: heat: DEFAULT: num_engine_workers: 2 heat_api: workers: 2 heat_api_cfn: workers: 2 heat_api_cloudwatch: workers: 2 pod: replicas: api: 1 cfn: 1 cloudwatch: 1 engine: 1 horizon_helm_values: pod: replicas: server: 1 ingress_nginx_helm_values: controller: config: worker-processes: 2 keystone_helm_values: pod: replicas: api: 1 kube_vip_address: 172.17.0.100 kube_vip_interface: '{{ ansible_facts[''default_ipv4''].interface }}' kubernetes_hostname: '{{ ansible_facts[''default_ipv4''].address }}' kubernetes_keepalived_interface: br-mgmt magnum_helm_values: conf: magnum: api: workers: 2 conductor: workers: 2 pod: replicas: api: 1 conductor: 1 magnum_image_disk_format: qcow2 magnum_images: '[ {{ _magnum_images[-1] }} ]' manila_helm_values: conf: manila: DEFAULT: osapi_share_workers: 2 pod: replicas: api: 1 scheduler: 1 molecule_scenario: aio neutron_helm_values: conf: neutron: DEFAULT: api_workers: 2 metadata_workers: 2 rpc_workers: 2 pod: replicas: rpc_server: 1 server: 1 nova_helm_values: conf: nova: DEFAULT: metadata_workers: 2 osapi_compute_workers: 2 conductor: workers: 2 scheduler: workers: 2 pod: replicas: api_metadata: 1 conductor: 1 novncproxy: 1 osapi: 1 scheduler: 1 spiceproxy: 1 octavia_helm_values: conf: octavia: controller_worker: workers: 2 octavia_api_uwsgi: uwsgi: processes: 2 pod: replicas: api: 1 housekeeping: 1 worker: 1 ovn_helm_values: conf: auto_bridge_add: br-ex: null pod: replicas: ovn_northd: 1 ovn_ovsdb_nb: 1 ovn_ovsdb_sb: 1 percona_xtradb_cluster_spec: allowUnsafeConfigurations: true haproxy: size: 1 pxc: size: 1 placement_helm_values: conf: placement_api_uwsgi: uwsgi: processes: 2 pod: replicas: api: 1 rook_ceph_cluster_radosgw_spec: dataPool: failureDomain: osd gateway: instances: 1 metadataPool: failureDomain: osd staffeln_helm_values: pod: replicas: api: 1 conductor: 1 valkey_helm_values: replica: replicaCount: 1 zuul: _inheritance_path: - '' - '' - '' - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: 3bce53d6fa5b46dca0ea7f3023ecd3f4 build_refs: - branch: main change: '3772' change_message: "feat(monitoring): add smartctl exporter for disk health monitoring\n\n## Summary\n\nAdd a new `smartctl_exporter` Ansible role that deploys the prometheus\nsmartctl_exporter as a DaemonSet on bare-metal nodes, plus a set of\nPrometheus alerting rules covering SMART/NVMe/SAS disk health. This\nenables early-warning monitoring of disk wearout, sector errors, NVMe\ncontroller warnings, SAS uncorrected errors, and overall SMART status.\n\n## Components\n\n- **`roles/smartctl_exporter/`**: DaemonSet deployment (privileged,\n scheduled onto bare-metal nodes only).\n- **ServiceMonitor**: registered in `kube_prometheus_stack` vars; the\n scrape config relabels `__meta_kubernetes_pod_node_name` to\n `instance` so every series is keyed by node, and drops the\n `pod`/`namespace`/`container`/`node`/`service` labels for\n cardinality.\n- **Alert rules** in `roles/kube_prometheus_stack/files/jsonnet/smartctl.libsonnet`\n \ (14 alerts, all with `runbook_url` and severity-aligned `for` durations):\n\n \ Health / firmware-declared failure:\n - `SmartctlDiskUnhealthy` (P2) \u2014 `smartctl_device_smart_status == 0`\n - `SmartctlDiskCriticalWarning` (P2) \u2014 NVMe critical_warning bitmap non-zero\n - `SmartctlDiskAvailableSpareLow` (P2) \u2014 NVMe available_spare \u2264 threshold\n - `SmartctlDiskAttributeFailing` (P2) \u2014 any SATA attribute's normalized\n `value` \u2264 per-drive `thresh` reported by firmware (covers any\n vendor-specific prefailure indicator without magic IDs)\n\n Wear and media errors:\n - `SmartctlDiskWearoutCritical` (P3) \u2014 NVMe `percentage_used > 90`\n - `SmartctlDiskWearoutWarning` (P4) \u2014 NVMe `percentage_used > 75`\n - `SmartctlDiskMediaErrorsGrowing` (P3) \u2014 NVMe `media_errors` increasing\n - `SmartctlDiskScsiUncorrectedErrors` (P3) \u2014 SAS read/write uncorrected\n error counters increasing\n \ - `SmartctlDiskScsiGrownDefectsGrowing` (P4) \u2014 SAS grown defect list\n \ increasing\n\n SATA sector errors:\n - `SmartctlDiskPendingSectors` (P3) \u2014 `Current_Pending_Sector` raw > 0\n - `SmartctlDiskUncorrectableSectors` (P3) \u2014 `Offline_Uncorrectable` raw > 0\n - `SmartctlDiskReallocatedSectorsGrowing` (P4) \u2014 rate-based, fires only\n on growth (not on a stable historical count)\n\n Operational:\n - `SmartctlDiskTemperatureHigh` (P4) \u2014 `temperature > 60\xB0C`\n - `SmartctlDiskSelfTestFailed` (P4) \u2014 self-test error log non-zero\n\n Note: there is no per-exporter \"down\" alert. The default\n kube-prometheus-stack `TargetDown` rule covers scrape-target outages\n uniformly across all exporters.\n\n- **Alertmanager inhibitions** (`roles/kube_prometheus_stack/vars/main.yml`):\n - `SmartctlDiskUnhealthy` suppresses every other smartctl alert on the\n same `(instance, device)`.\n \ - `SmartctlDiskCriticalWarning` and `SmartctlDiskAvailableSpareLow`\n \ suppress the lower-severity NVMe-specific alerts on the same disk.\n \ - `SmartctlDiskWearoutCritical` suppresses\n `SmartctlDiskWearoutWarning`.\n\n- **Documentation**: per-alert runbook entries added under\n `doc/source/admin/monitoring.rst`.\n\n- **Tests**: positive and negative `promtool` test cases for every\n alert in `roles/kube_prometheus_stack/files/jsonnet/tests.yml`.\n\n- Wired into `playbooks/monitoring.yml`.\n" change_url: https://github.com/vexxhost/atmosphere/pull/3772 commit_id: abca58f956837fe702574c78b0d022a0580e71c3 patchset: abca58f956837fe702574c78b0d022a0580e71c3 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null buildset: f4d1cd9f5757488fa0d116f80abc99df buildset_refs: - branch: main change: '3772' change_message: "feat(monitoring): add smartctl exporter for disk health monitoring\n\n## Summary\n\nAdd a new `smartctl_exporter` Ansible role that deploys the prometheus\nsmartctl_exporter as a DaemonSet on bare-metal nodes, plus a set of\nPrometheus alerting rules covering SMART/NVMe/SAS disk health. This\nenables early-warning monitoring of disk wearout, sector errors, NVMe\ncontroller warnings, SAS uncorrected errors, and overall SMART status.\n\n## Components\n\n- **`roles/smartctl_exporter/`**: DaemonSet deployment (privileged,\n scheduled onto bare-metal nodes only).\n- **ServiceMonitor**: registered in `kube_prometheus_stack` vars; the\n scrape config relabels `__meta_kubernetes_pod_node_name` to\n `instance` so every series is keyed by node, and drops the\n `pod`/`namespace`/`container`/`node`/`service` labels for\n cardinality.\n- **Alert rules** in `roles/kube_prometheus_stack/files/jsonnet/smartctl.libsonnet`\n \ (14 alerts, all with `runbook_url` and severity-aligned `for` durations):\n\n \ Health / firmware-declared failure:\n - `SmartctlDiskUnhealthy` (P2) \u2014 `smartctl_device_smart_status == 0`\n - `SmartctlDiskCriticalWarning` (P2) \u2014 NVMe critical_warning bitmap non-zero\n - `SmartctlDiskAvailableSpareLow` (P2) \u2014 NVMe available_spare \u2264 threshold\n - `SmartctlDiskAttributeFailing` (P2) \u2014 any SATA attribute's normalized\n `value` \u2264 per-drive `thresh` reported by firmware (covers any\n vendor-specific prefailure indicator without magic IDs)\n\n Wear and media errors:\n - `SmartctlDiskWearoutCritical` (P3) \u2014 NVMe `percentage_used > 90`\n - `SmartctlDiskWearoutWarning` (P4) \u2014 NVMe `percentage_used > 75`\n - `SmartctlDiskMediaErrorsGrowing` (P3) \u2014 NVMe `media_errors` increasing\n - `SmartctlDiskScsiUncorrectedErrors` (P3) \u2014 SAS read/write uncorrected\n error counters increasing\n \ - `SmartctlDiskScsiGrownDefectsGrowing` (P4) \u2014 SAS grown defect list\n \ increasing\n\n SATA sector errors:\n - `SmartctlDiskPendingSectors` (P3) \u2014 `Current_Pending_Sector` raw > 0\n - `SmartctlDiskUncorrectableSectors` (P3) \u2014 `Offline_Uncorrectable` raw > 0\n - `SmartctlDiskReallocatedSectorsGrowing` (P4) \u2014 rate-based, fires only\n on growth (not on a stable historical count)\n\n Operational:\n - `SmartctlDiskTemperatureHigh` (P4) \u2014 `temperature > 60\xB0C`\n - `SmartctlDiskSelfTestFailed` (P4) \u2014 self-test error log non-zero\n\n Note: there is no per-exporter \"down\" alert. The default\n kube-prometheus-stack `TargetDown` rule covers scrape-target outages\n uniformly across all exporters.\n\n- **Alertmanager inhibitions** (`roles/kube_prometheus_stack/vars/main.yml`):\n - `SmartctlDiskUnhealthy` suppresses every other smartctl alert on the\n same `(instance, device)`.\n \ - `SmartctlDiskCriticalWarning` and `SmartctlDiskAvailableSpareLow`\n \ suppress the lower-severity NVMe-specific alerts on the same disk.\n \ - `SmartctlDiskWearoutCritical` suppresses\n `SmartctlDiskWearoutWarning`.\n\n- **Documentation**: per-alert runbook entries added under\n `doc/source/admin/monitoring.rst`.\n\n- **Tests**: positive and negative `promtool` test cases for every\n alert in `roles/kube_prometheus_stack/files/jsonnet/tests.yml`.\n\n- Wired into `playbooks/monitoring.yml`.\n" change_url: https://github.com/vexxhost/atmosphere/pull/3772 commit_id: abca58f956837fe702574c78b0d022a0580e71c3 patchset: abca58f956837fe702574c78b0d022a0580e71c3 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null change: '3772' change_message: "feat(monitoring): add smartctl exporter for disk health monitoring\n\n## Summary\n\nAdd a new `smartctl_exporter` Ansible role that deploys the prometheus\nsmartctl_exporter as a DaemonSet on bare-metal nodes, plus a set of\nPrometheus alerting rules covering SMART/NVMe/SAS disk health. This\nenables early-warning monitoring of disk wearout, sector errors, NVMe\ncontroller warnings, SAS uncorrected errors, and overall SMART status.\n\n## Components\n\n- **`roles/smartctl_exporter/`**: DaemonSet deployment (privileged,\n scheduled onto bare-metal nodes only).\n- **ServiceMonitor**: registered in `kube_prometheus_stack` vars; the\n scrape config relabels `__meta_kubernetes_pod_node_name` to\n `instance` so every series is keyed by node, and drops the\n `pod`/`namespace`/`container`/`node`/`service` labels for\n cardinality.\n- **Alert rules** in `roles/kube_prometheus_stack/files/jsonnet/smartctl.libsonnet`\n \ (14 alerts, all with `runbook_url` and severity-aligned `for` durations):\n\n \ Health / firmware-declared failure:\n - `SmartctlDiskUnhealthy` (P2) \u2014 `smartctl_device_smart_status == 0`\n - `SmartctlDiskCriticalWarning` (P2) \u2014 NVMe critical_warning bitmap non-zero\n - `SmartctlDiskAvailableSpareLow` (P2) \u2014 NVMe available_spare \u2264 threshold\n - `SmartctlDiskAttributeFailing` (P2) \u2014 any SATA attribute's normalized\n `value` \u2264 per-drive `thresh` reported by firmware (covers any\n vendor-specific prefailure indicator without magic IDs)\n\n Wear and media errors:\n - `SmartctlDiskWearoutCritical` (P3) \u2014 NVMe `percentage_used > 90`\n - `SmartctlDiskWearoutWarning` (P4) \u2014 NVMe `percentage_used > 75`\n - `SmartctlDiskMediaErrorsGrowing` (P3) \u2014 NVMe `media_errors` increasing\n - `SmartctlDiskScsiUncorrectedErrors` (P3) \u2014 SAS read/write uncorrected\n error counters increasing\n - `SmartctlDiskScsiGrownDefectsGrowing` (P4) \u2014 SAS grown defect list\n \ increasing\n\n SATA sector errors:\n - `SmartctlDiskPendingSectors` (P3) \u2014 `Current_Pending_Sector` raw > 0\n - `SmartctlDiskUncorrectableSectors` (P3) \u2014 `Offline_Uncorrectable` raw > 0\n - `SmartctlDiskReallocatedSectorsGrowing` (P4) \u2014 rate-based, fires only\n on growth (not on a stable historical count)\n\n Operational:\n - `SmartctlDiskTemperatureHigh` (P4) \u2014 `temperature > 60\xB0C`\n - `SmartctlDiskSelfTestFailed` (P4) \u2014 self-test error log non-zero\n\n Note: there is no per-exporter \"down\" alert. The default\n \ kube-prometheus-stack `TargetDown` rule covers scrape-target outages\n uniformly across all exporters.\n\n- **Alertmanager inhibitions** (`roles/kube_prometheus_stack/vars/main.yml`):\n \ - `SmartctlDiskUnhealthy` suppresses every other smartctl alert on the\n \ same `(instance, device)`.\n - `SmartctlDiskCriticalWarning` and `SmartctlDiskAvailableSpareLow`\n \ suppress the lower-severity NVMe-specific alerts on the same disk.\n - `SmartctlDiskWearoutCritical` suppresses\n `SmartctlDiskWearoutWarning`.\n\n- **Documentation**: per-alert runbook entries added under\n `doc/source/admin/monitoring.rst`.\n\n- **Tests**: positive and negative `promtool` test cases for every\n alert in `roles/kube_prometheus_stack/files/jsonnet/tests.yml`.\n\n- Wired into `playbooks/monitoring.yml`.\n" change_url: https://github.com/vexxhost/atmosphere/pull/3772 child_jobs: [] commit_id: abca58f956837fe702574c78b0d022a0580e71c3 event_id: ea8bbc20-533e-11f1-8035-dcdfc53d67dd executor: hostname: 2d72f0692154 inventory_file: /var/lib/zuul/builds/3bce53d6fa5b46dca0ea7f3023ecd3f4/ansible/inventory.yaml log_root: /var/lib/zuul/builds/3bce53d6fa5b46dca0ea7f3023ecd3f4/work/logs result_data_file: /var/lib/zuul/builds/3bce53d6fa5b46dca0ea7f3023ecd3f4/work/results.json src_root: /var/lib/zuul/builds/3bce53d6fa5b46dca0ea7f3023ecd3f4/work/src work_root: /var/lib/zuul/builds/3bce53d6fa5b46dca0ea7f3023ecd3f4/work include_vars: [] items: - branch: main change: '3772' change_message: "feat(monitoring): add smartctl exporter for disk health monitoring\n\n## Summary\n\nAdd a new `smartctl_exporter` Ansible role that deploys the prometheus\nsmartctl_exporter as a DaemonSet on bare-metal nodes, plus a set of\nPrometheus alerting rules covering SMART/NVMe/SAS disk health. This\nenables early-warning monitoring of disk wearout, sector errors, NVMe\ncontroller warnings, SAS uncorrected errors, and overall SMART status.\n\n## Components\n\n- **`roles/smartctl_exporter/`**: DaemonSet deployment (privileged,\n scheduled onto bare-metal nodes only).\n- **ServiceMonitor**: registered in `kube_prometheus_stack` vars; the\n scrape config relabels `__meta_kubernetes_pod_node_name` to\n `instance` so every series is keyed by node, and drops the\n `pod`/`namespace`/`container`/`node`/`service` labels for\n cardinality.\n- **Alert rules** in `roles/kube_prometheus_stack/files/jsonnet/smartctl.libsonnet`\n \ (14 alerts, all with `runbook_url` and severity-aligned `for` durations):\n\n \ Health / firmware-declared failure:\n - `SmartctlDiskUnhealthy` (P2) \u2014 `smartctl_device_smart_status == 0`\n - `SmartctlDiskCriticalWarning` (P2) \u2014 NVMe critical_warning bitmap non-zero\n - `SmartctlDiskAvailableSpareLow` (P2) \u2014 NVMe available_spare \u2264 threshold\n - `SmartctlDiskAttributeFailing` (P2) \u2014 any SATA attribute's normalized\n `value` \u2264 per-drive `thresh` reported by firmware (covers any\n vendor-specific prefailure indicator without magic IDs)\n\n Wear and media errors:\n - `SmartctlDiskWearoutCritical` (P3) \u2014 NVMe `percentage_used > 90`\n - `SmartctlDiskWearoutWarning` (P4) \u2014 NVMe `percentage_used > 75`\n - `SmartctlDiskMediaErrorsGrowing` (P3) \u2014 NVMe `media_errors` increasing\n - `SmartctlDiskScsiUncorrectedErrors` (P3) \u2014 SAS read/write uncorrected\n error counters increasing\n \ - `SmartctlDiskScsiGrownDefectsGrowing` (P4) \u2014 SAS grown defect list\n \ increasing\n\n SATA sector errors:\n - `SmartctlDiskPendingSectors` (P3) \u2014 `Current_Pending_Sector` raw > 0\n - `SmartctlDiskUncorrectableSectors` (P3) \u2014 `Offline_Uncorrectable` raw > 0\n - `SmartctlDiskReallocatedSectorsGrowing` (P4) \u2014 rate-based, fires only\n on growth (not on a stable historical count)\n\n Operational:\n - `SmartctlDiskTemperatureHigh` (P4) \u2014 `temperature > 60\xB0C`\n - `SmartctlDiskSelfTestFailed` (P4) \u2014 self-test error log non-zero\n\n Note: there is no per-exporter \"down\" alert. The default\n kube-prometheus-stack `TargetDown` rule covers scrape-target outages\n uniformly across all exporters.\n\n- **Alertmanager inhibitions** (`roles/kube_prometheus_stack/vars/main.yml`):\n - `SmartctlDiskUnhealthy` suppresses every other smartctl alert on the\n same `(instance, device)`.\n \ - `SmartctlDiskCriticalWarning` and `SmartctlDiskAvailableSpareLow`\n \ suppress the lower-severity NVMe-specific alerts on the same disk.\n \ - `SmartctlDiskWearoutCritical` suppresses\n `SmartctlDiskWearoutWarning`.\n\n- **Documentation**: per-alert runbook entries added under\n `doc/source/admin/monitoring.rst`.\n\n- **Tests**: positive and negative `promtool` test cases for every\n alert in `roles/kube_prometheus_stack/files/jsonnet/tests.yml`.\n\n- Wired into `playbooks/monitoring.yml`.\n" change_url: https://github.com/vexxhost/atmosphere/pull/3772 commit_id: abca58f956837fe702574c78b0d022a0580e71c3 patchset: abca58f956837fe702574c78b0d022a0580e71c3 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere topic: null job: atmosphere-molecule-aio-openvswitch jobtags: [] max_attempts: 3 message: ZmVhdChtb25pdG9yaW5nKTogYWRkIHNtYXJ0Y3RsIGV4cG9ydGVyIGZvciBkaXNrIGhlYWx0aCBtb25pdG9yaW5nCgojIyBTdW1tYXJ5CgpBZGQgYSBuZXcgYHNtYXJ0Y3RsX2V4cG9ydGVyYCBBbnNpYmxlIHJvbGUgdGhhdCBkZXBsb3lzIHRoZSBwcm9tZXRoZXVzCnNtYXJ0Y3RsX2V4cG9ydGVyIGFzIGEgRGFlbW9uU2V0IG9uIGJhcmUtbWV0YWwgbm9kZXMsIHBsdXMgYSBzZXQgb2YKUHJvbWV0aGV1cyBhbGVydGluZyBydWxlcyBjb3ZlcmluZyBTTUFSVC9OVk1lL1NBUyBkaXNrIGhlYWx0aC4gVGhpcwplbmFibGVzIGVhcmx5LXdhcm5pbmcgbW9uaXRvcmluZyBvZiBkaXNrIHdlYXJvdXQsIHNlY3RvciBlcnJvcnMsIE5WTWUKY29udHJvbGxlciB3YXJuaW5ncywgU0FTIHVuY29ycmVjdGVkIGVycm9ycywgYW5kIG92ZXJhbGwgU01BUlQgc3RhdHVzLgoKIyMgQ29tcG9uZW50cwoKLSAqKmByb2xlcy9zbWFydGN0bF9leHBvcnRlci9gKio6IERhZW1vblNldCBkZXBsb3ltZW50IChwcml2aWxlZ2VkLAogIHNjaGVkdWxlZCBvbnRvIGJhcmUtbWV0YWwgbm9kZXMgb25seSkuCi0gKipTZXJ2aWNlTW9uaXRvcioqOiByZWdpc3RlcmVkIGluIGBrdWJlX3Byb21ldGhldXNfc3RhY2tgIHZhcnM7IHRoZQogIHNjcmFwZSBjb25maWcgcmVsYWJlbHMgYF9fbWV0YV9rdWJlcm5ldGVzX3BvZF9ub2RlX25hbWVgIHRvCiAgYGluc3RhbmNlYCBzbyBldmVyeSBzZXJpZXMgaXMga2V5ZWQgYnkgbm9kZSwgYW5kIGRyb3BzIHRoZQogIGBwb2RgL2BuYW1lc3BhY2VgL2Bjb250YWluZXJgL2Bub2RlYC9gc2VydmljZWAgbGFiZWxzIGZvcgogIGNhcmRpbmFsaXR5LgotICoqQWxlcnQgcnVsZXMqKiBpbiBgcm9sZXMva3ViZV9wcm9tZXRoZXVzX3N0YWNrL2ZpbGVzL2pzb25uZXQvc21hcnRjdGwubGlic29ubmV0YAogICgxNCBhbGVydHMsIGFsbCB3aXRoIGBydW5ib29rX3VybGAgYW5kIHNldmVyaXR5LWFsaWduZWQgYGZvcmAgZHVyYXRpb25zKToKCiAgSGVhbHRoIC8gZmlybXdhcmUtZGVjbGFyZWQgZmFpbHVyZToKICAtIGBTbWFydGN0bERpc2tVbmhlYWx0aHlgIChQMikg4oCUIGBzbWFydGN0bF9kZXZpY2Vfc21hcnRfc3RhdHVzID09IDBgCiAgLSBgU21hcnRjdGxEaXNrQ3JpdGljYWxXYXJuaW5nYCAoUDIpIOKAlCBOVk1lIGNyaXRpY2FsX3dhcm5pbmcgYml0bWFwIG5vbi16ZXJvCiAgLSBgU21hcnRjdGxEaXNrQXZhaWxhYmxlU3BhcmVMb3dgIChQMikg4oCUIE5WTWUgYXZhaWxhYmxlX3NwYXJlIOKJpCB0aHJlc2hvbGQKICAtIGBTbWFydGN0bERpc2tBdHRyaWJ1dGVGYWlsaW5nYCAoUDIpIOKAlCBhbnkgU0FUQSBhdHRyaWJ1dGUncyBub3JtYWxpemVkCiAgICBgdmFsdWVgIOKJpCBwZXItZHJpdmUgYHRocmVzaGAgcmVwb3J0ZWQgYnkgZmlybXdhcmUgKGNvdmVycyBhbnkKICAgIHZlbmRvci1zcGVjaWZpYyBwcmVmYWlsdXJlIGluZGljYXRvciB3aXRob3V0IG1hZ2ljIElEcykKCiAgV2VhciBhbmQgbWVkaWEgZXJyb3JzOgogIC0gYFNtYXJ0Y3RsRGlza1dlYXJvdXRDcml0aWNhbGAgKFAzKSDigJQgTlZNZSBgcGVyY2VudGFnZV91c2VkID4gOTBgCiAgLSBgU21hcnRjdGxEaXNrV2Vhcm91dFdhcm5pbmdgIChQNCkg4oCUIE5WTWUgYHBlcmNlbnRhZ2VfdXNlZCA+IDc1YAogIC0gYFNtYXJ0Y3RsRGlza01lZGlhRXJyb3JzR3Jvd2luZ2AgKFAzKSDigJQgTlZNZSBgbWVkaWFfZXJyb3JzYCBpbmNyZWFzaW5nCiAgLSBgU21hcnRjdGxEaXNrU2NzaVVuY29ycmVjdGVkRXJyb3JzYCAoUDMpIOKAlCBTQVMgcmVhZC93cml0ZSB1bmNvcnJlY3RlZAogICAgZXJyb3IgY291bnRlcnMgaW5jcmVhc2luZwogIC0gYFNtYXJ0Y3RsRGlza1Njc2lHcm93bkRlZmVjdHNHcm93aW5nYCAoUDQpIOKAlCBTQVMgZ3Jvd24gZGVmZWN0IGxpc3QKICAgIGluY3JlYXNpbmcKCiAgU0FUQSBzZWN0b3IgZXJyb3JzOgogIC0gYFNtYXJ0Y3RsRGlza1BlbmRpbmdTZWN0b3JzYCAoUDMpIOKAlCBgQ3VycmVudF9QZW5kaW5nX1NlY3RvcmAgcmF3ID4gMAogIC0gYFNtYXJ0Y3RsRGlza1VuY29ycmVjdGFibGVTZWN0b3JzYCAoUDMpIOKAlCBgT2ZmbGluZV9VbmNvcnJlY3RhYmxlYCByYXcgPiAwCiAgLSBgU21hcnRjdGxEaXNrUmVhbGxvY2F0ZWRTZWN0b3JzR3Jvd2luZ2AgKFA0KSDigJQgcmF0ZS1iYXNlZCwgZmlyZXMgb25seQogICAgb24gZ3Jvd3RoIChub3Qgb24gYSBzdGFibGUgaGlzdG9yaWNhbCBjb3VudCkKCiAgT3BlcmF0aW9uYWw6CiAgLSBgU21hcnRjdGxEaXNrVGVtcGVyYXR1cmVIaWdoYCAoUDQpIOKAlCBgdGVtcGVyYXR1cmUgPiA2MMKwQ2AKICAtIGBTbWFydGN0bERpc2tTZWxmVGVzdEZhaWxlZGAgKFA0KSDigJQgc2VsZi10ZXN0IGVycm9yIGxvZyBub24temVybwoKICBOb3RlOiB0aGVyZSBpcyBubyBwZXItZXhwb3J0ZXIgImRvd24iIGFsZXJ0LiBUaGUgZGVmYXVsdAogIGt1YmUtcHJvbWV0aGV1cy1zdGFjayBgVGFyZ2V0RG93bmAgcnVsZSBjb3ZlcnMgc2NyYXBlLXRhcmdldCBvdXRhZ2VzCiAgdW5pZm9ybWx5IGFjcm9zcyBhbGwgZXhwb3J0ZXJzLgoKLSAqKkFsZXJ0bWFuYWdlciBpbmhpYml0aW9ucyoqIChgcm9sZXMva3ViZV9wcm9tZXRoZXVzX3N0YWNrL3ZhcnMvbWFpbi55bWxgKToKICAtIGBTbWFydGN0bERpc2tVbmhlYWx0aHlgIHN1cHByZXNzZXMgZXZlcnkgb3RoZXIgc21hcnRjdGwgYWxlcnQgb24gdGhlCiAgICBzYW1lIGAoaW5zdGFuY2UsIGRldmljZSlgLgogIC0gYFNtYXJ0Y3RsRGlza0NyaXRpY2FsV2FybmluZ2AgYW5kIGBTbWFydGN0bERpc2tBdmFpbGFibGVTcGFyZUxvd2AKICAgIHN1cHByZXNzIHRoZSBsb3dlci1zZXZlcml0eSBOVk1lLXNwZWNpZmljIGFsZXJ0cyBvbiB0aGUgc2FtZSBkaXNrLgogIC0gYFNtYXJ0Y3RsRGlza1dlYXJvdXRDcml0aWNhbGAgc3VwcHJlc3NlcwogICAgYFNtYXJ0Y3RsRGlza1dlYXJvdXRXYXJuaW5nYC4KCi0gKipEb2N1bWVudGF0aW9uKio6IHBlci1hbGVydCBydW5ib29rIGVudHJpZXMgYWRkZWQgdW5kZXIKICBgZG9jL3NvdXJjZS9hZG1pbi9tb25pdG9yaW5nLnJzdGAuCgotICoqVGVzdHMqKjogcG9zaXRpdmUgYW5kIG5lZ2F0aXZlIGBwcm9tdG9vbGAgdGVzdCBjYXNlcyBmb3IgZXZlcnkKICBhbGVydCBpbiBgcm9sZXMva3ViZV9wcm9tZXRoZXVzX3N0YWNrL2ZpbGVzL2pzb25uZXQvdGVzdHMueW1sYC4KCi0gV2lyZWQgaW50byBgcGxheWJvb2tzL21vbml0b3JpbmcueW1sYC4K patchset: abca58f956837fe702574c78b0d022a0580e71c3 pipeline: check playbook_context: playbook_projects: trusted/project_0/github.com/vexxhost/zuul-config: canonical_name: github.com/vexxhost/zuul-config checkout: main commit: 298983cd1253e6833abdb49d87d912527e0e6597 trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 79fe3eb1d01f8ac5739b0b7bc4759c407b6e248d trusted/project_2/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: a6e68243e02ef030ce5e75f8b67630880c475f33 untrusted/project_0/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: a6e68243e02ef030ce5e75f8b67630880c475f33 untrusted/project_1/github.com/vexxhost/zuul-config: canonical_name: github.com/vexxhost/zuul-config checkout: main commit: 298983cd1253e6833abdb49d87d912527e0e6597 untrusted/project_2/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 79fe3eb1d01f8ac5739b0b7bc4759c407b6e248d untrusted/project_3/github.com/vexxhost/atmosphere: canonical_name: github.com/vexxhost/atmosphere checkout: main commit: abca58f956837fe702574c78b0d022a0580e71c3 untrusted/project_4/opendev.org/openstack/openstack-helm: canonical_name: opendev.org/openstack/openstack-helm checkout: master commit: ebd6507b8b84365fe43c389f9571959da7b0826c playbooks: - path: untrusted/project_0/github.com/vexxhost/zuul-jobs/playbooks/molecule/run.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_1/zuul-jobs/roles - checkout: main checkout_description: playbook branch link_name: ansible/playbook_0/role_2/zuul-jobs link_target: untrusted/project_0/github.com/vexxhost/zuul-jobs role_path: ansible/playbook_0/role_2/zuul-jobs/roles post_playbooks: - path: untrusted/project_3/github.com/vexxhost/atmosphere/test-playbooks/molecule/post.yml roles: - checkout: main checkout_description: playbook branch link_name: ansible/post_playbook_0/role_0/atmosphere link_target: untrusted/project_3/github.com/vexxhost/atmosphere role_path: ansible/post_playbook_0/role_0/atmosphere/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_1/openstack-helm link_target: untrusted/project_4/opendev.org/openstack/openstack-helm role_path: ansible/post_playbook_0/role_1/openstack-helm/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_3/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_0/role_3/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_0/role_4/zuul-jobs link_target: untrusted/project_0/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_0/role_4/zuul-jobs/roles - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/post.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_1/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_1/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_1/role_2/zuul-jobs/roles - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/post-logs.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_2/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_2/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_2/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_2/role_2/zuul-jobs/roles pre_playbooks: - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/pre.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_0/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/pre_playbook_0/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/pre_playbook_0/role_2/zuul-jobs/roles - path: untrusted/project_0/github.com/vexxhost/zuul-jobs/playbooks/molecule/pre.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_1/role_1/zuul-jobs/roles - checkout: main checkout_description: playbook branch link_name: ansible/pre_playbook_1/role_2/zuul-jobs link_target: untrusted/project_0/github.com/vexxhost/zuul-jobs role_path: ansible/pre_playbook_1/role_2/zuul-jobs/roles - path: untrusted/project_3/github.com/vexxhost/atmosphere/test-playbooks/molecule/pre.yml roles: - checkout: main checkout_description: playbook branch link_name: ansible/pre_playbook_2/role_0/atmosphere link_target: untrusted/project_3/github.com/vexxhost/atmosphere role_path: ansible/pre_playbook_2/role_0/atmosphere/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_2/role_1/openstack-helm link_target: untrusted/project_4/opendev.org/openstack/openstack-helm role_path: ansible/pre_playbook_2/role_1/openstack-helm/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_2/role_3/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_2/role_3/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/pre_playbook_2/role_4/zuul-jobs link_target: untrusted/project_0/github.com/vexxhost/zuul-jobs role_path: ansible/pre_playbook_2/role_4/zuul-jobs/roles post_review: false post_timeout: null pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere name: vexxhost/atmosphere short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere projects: github.com/vexxhost/atmosphere: canonical_hostname: github.com canonical_name: github.com/vexxhost/atmosphere checkout: main checkout_description: zuul branch commit: abca58f956837fe702574c78b0d022a0580e71c3 name: vexxhost/atmosphere required: false short_name: atmosphere src_dir: src/github.com/vexxhost/atmosphere ref: refs/pull/3772/head resources: {} tenant: oss timeout: 7200 topic: null voting: true