all: children: tempest: hosts: controller: null zuul_unreachable: hosts: {} hosts: controller: ansible_connection: ssh ansible_host: 199.204.45.33 ansible_port: 22 ansible_python_interpreter: auto ansible_user: zuul configure_swap_size: 8192 devstack_local_conf: post-config: $NEUTRON_CONF: DEFAULT: global_physnet_mtu: '{{ external_bridge_mtu }}' /etc/magnum/magnum.conf: cluster_template: kubernetes_allowed_network_drivers: calico,cilium kubernetes_default_network_driver: calico nova_client: api_version: 2.15 /etc/manila/manila.conf: generic: connect_share_server_to_tenant_network: true driver_handles_share_servers: true devstack_localrc: ADMIN_PASSWORD: secretadmin DATABASE_PASSWORD: secretdatabase DEBUG_LIBVIRT_COREDUMPS: true DISABLE_AMP_IMAGE_BUILD: true ENABLE_SYSCTL_MEM_TUNING: true ENABLE_SYSCTL_NET_TUNING: true ENABLE_ZSWAP: true ERROR_ON_CLONE: true FIXED_RANGE: 10.1.0.0/20 FLOATING_RANGE: 172.24.5.0/24 GIT_BASE: https://github.com HOST_IP: '{{ hostvars[''controller''][''nodepool''][''private_ipv4''] }}' IPV4_ADDRS_SAFE_TO_USE: 10.1.0.0/20 LIBVIRT_TYPE: '{{ devstack_libvirt_type | default("qemu") }}' LOGFILE: /opt/stack/logs/devstacklog.txt LOG_COLOR: false MAGNUM_GUEST_IMAGE_URL: '{{ image_url }}' MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS: snapshot_support=True create_share_from_snapshot_support=True MANILA_ENABLED_BACKENDS: generic MANILA_USE_SERVICE_INSTANCE_PASSWORD: true NETWORK_GATEWAY: 10.1.0.1 NOVA_LIBVIRT_TB_CACHE_SIZE: 128 NOVA_VNC_ENABLED: true OCTAVIA_NODE: api OVN_DBS_LOG_LEVEL: dbg PUBLIC_BRIDGE_MTU: '{{ external_bridge_mtu }}' PUBLIC_NETWORK_GATEWAY: 172.24.5.1 RABBIT_PASSWORD: secretrabbit SERVICE_HOST: '{{ hostvars[''controller''][''nodepool''][''private_ipv4''] }}' SERVICE_PASSWORD: secretservice SWIFT_HASH: 1234123412341234 SWIFT_REPLICAS: 1 SWIFT_START_ALL_SERVICES: false VERBOSE: true VERBOSE_NO_TIMESTAMP: true devstack_plugins: barbican: https://github.com/openstack/barbican magnum: https://review.opendev.org/openstack/magnum magnum-cluster-api: https://github.com/vexxhost/magnum-cluster-api manila: https://github.com/openstack/manila octavia: https://github.com/openstack/octavia ovn-octavia-provider: https://github.com/openstack/ovn-octavia-provider devstack_services: base: false c-api: true c-bak: true c-sch: true c-vol: true dstat: false etcd3: true file_tracker: true g-api: true horizon: false key: true memory_tracker: true mysql: true n-api: true n-api-meta: true n-cond: true n-cpu: true n-novnc: true n-sch: true o-api: true o-da: true o-hk: true octavia: true openstack-cli-server: true ovn-controller: true ovn-northd: true ovs-vswitchd: true ovsdb-server: true placement-api: true q-ovn-agent: true q-svc: true rabbit: true s-account: false s-container: false s-object: false s-proxy: false tempest: false tls-proxy: true extensions_to_txt: auto: true conf: true localrc: true log: true stackenv: true image_url: https://github.com/vexxhost/capo-image-elements/releases/latest/download/ubuntu-22.04-{{ kube_tag }}.qcow2 kube_tag: v1.34.7 network_driver: calico nodepool: az: nova cloud: public external_id: 28e8c9af-d12b-461b-ad9e-36bb794109fa host_id: 7b8a4cf3090e32af994f4889a62f1003a5f74592d8295d185d35c3b9 interface_ip: 199.204.45.33 label: ubuntu-noble-16 node_properties: {} private_ipv4: 199.204.45.33 private_ipv6: null provider: yul1 public_ipv4: 199.204.45.33 public_ipv6: 2604:e100:1:0:f816:3eff:feb6:b43f region: ca-ymq-1 slot: null zuul_copy_output: /etc/ceph: logs /etc/glusterfs/glusterd.vol: logs /etc/libvirt: logs /etc/lvm: logs /etc/resolv.conf: logs /etc/sudoers: logs /etc/sudoers.d: logs /var/log/ceph: logs /var/log/glusterfs: logs /var/log/libvirt: logs /var/log/mysql: logs /var/log/openvswitch: logs /var/log/postgresql: logs /var/log/rabbitmq: logs /var/log/unbound.log: logs '{{ devstack_conf_dir }}/.localrc.auto': logs '{{ devstack_conf_dir }}/.stackenv': logs '{{ devstack_conf_dir }}/local.conf': logs '{{ devstack_conf_dir }}/localrc': logs '{{ devstack_full_log}}': logs '{{ devstack_log_dir }}/atop': logs '{{ devstack_log_dir }}/devstacklog.txt': logs '{{ devstack_log_dir }}/devstacklog.txt.summary': logs '{{ devstack_log_dir }}/dstat-csv.log': logs '{{ devstack_log_dir }}/qemu.coredump': logs '{{ devstack_log_dir }}/tcpdump.pcap': logs '{{ devstack_log_dir }}/worlddump-latest.txt': logs '{{ stage_dir }}/apache': logs '{{ stage_dir }}/apache_config': logs '{{ stage_dir }}/audit.log': logs '{{ stage_dir }}/core': logs '{{ stage_dir }}/deprecations.log': logs '{{ stage_dir }}/df.txt': logs '{{ stage_dir }}/dpkg-l.txt': logs '{{ stage_dir }}/etc': logs '{{ stage_dir }}/iptables.txt': logs '{{ stage_dir }}/listen53.txt': logs '{{ stage_dir }}/mount.txt': logs '{{ stage_dir }}/performance.json': logs '{{ stage_dir }}/pip2-freeze.txt': logs '{{ stage_dir }}/pip3-freeze.txt': logs '{{ stage_dir }}/rpm-qa.txt': logs '{{ stage_dir }}/services.txt': logs '{{ stage_dir }}/verify_tempest_conf.log': logs zuul_node: az: nova cloud: public external_id: 28e8c9af-d12b-461b-ad9e-36bb794109fa host_id: 7b8a4cf3090e32af994f4889a62f1003a5f74592d8295d185d35c3b9 interface_ip: 199.204.45.33 label: ubuntu-noble-16 node_properties: {} private_ipv4: 199.204.45.33 private_ipv6: null provider: yul1 public_ipv4: 199.204.45.33 public_ipv6: 2604:e100:1:0:f816:3eff:feb6:b43f region: ca-ymq-1 slot: null uuid: null vars: configure_swap_size: 8192 devstack_local_conf: post-config: $NEUTRON_CONF: DEFAULT: global_physnet_mtu: '{{ external_bridge_mtu }}' /etc/magnum/magnum.conf: cluster_template: kubernetes_allowed_network_drivers: calico,cilium kubernetes_default_network_driver: calico nova_client: api_version: 2.15 /etc/manila/manila.conf: generic: connect_share_server_to_tenant_network: true driver_handles_share_servers: true devstack_localrc: ADMIN_PASSWORD: secretadmin DATABASE_PASSWORD: secretdatabase DEBUG_LIBVIRT_COREDUMPS: true DISABLE_AMP_IMAGE_BUILD: true ENABLE_SYSCTL_MEM_TUNING: true ENABLE_SYSCTL_NET_TUNING: true ENABLE_ZSWAP: true ERROR_ON_CLONE: true FIXED_RANGE: 10.1.0.0/20 FLOATING_RANGE: 172.24.5.0/24 GIT_BASE: https://github.com HOST_IP: '{{ hostvars[''controller''][''nodepool''][''private_ipv4''] }}' IPV4_ADDRS_SAFE_TO_USE: 10.1.0.0/20 LIBVIRT_TYPE: '{{ devstack_libvirt_type | default("qemu") }}' LOGFILE: /opt/stack/logs/devstacklog.txt LOG_COLOR: false MAGNUM_GUEST_IMAGE_URL: '{{ image_url }}' MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS: snapshot_support=True create_share_from_snapshot_support=True MANILA_ENABLED_BACKENDS: generic MANILA_USE_SERVICE_INSTANCE_PASSWORD: true NETWORK_GATEWAY: 10.1.0.1 NOVA_LIBVIRT_TB_CACHE_SIZE: 128 NOVA_VNC_ENABLED: true OCTAVIA_NODE: api OVN_DBS_LOG_LEVEL: dbg PUBLIC_BRIDGE_MTU: '{{ external_bridge_mtu }}' PUBLIC_NETWORK_GATEWAY: 172.24.5.1 RABBIT_PASSWORD: secretrabbit SERVICE_HOST: '{{ hostvars[''controller''][''nodepool''][''private_ipv4''] }}' SERVICE_PASSWORD: secretservice SWIFT_HASH: 1234123412341234 SWIFT_REPLICAS: 1 SWIFT_START_ALL_SERVICES: false VERBOSE: true VERBOSE_NO_TIMESTAMP: true devstack_plugins: barbican: https://github.com/openstack/barbican magnum: https://review.opendev.org/openstack/magnum magnum-cluster-api: https://github.com/vexxhost/magnum-cluster-api manila: https://github.com/openstack/manila octavia: https://github.com/openstack/octavia ovn-octavia-provider: https://github.com/openstack/ovn-octavia-provider devstack_services: base: false c-api: true c-bak: true c-sch: true c-vol: true dstat: false etcd3: true file_tracker: true g-api: true horizon: false key: true memory_tracker: true mysql: true n-api: true n-api-meta: true n-cond: true n-cpu: true n-novnc: true n-sch: true o-api: true o-da: true o-hk: true octavia: true openstack-cli-server: true ovn-controller: true ovn-northd: true ovs-vswitchd: true ovsdb-server: true placement-api: true q-ovn-agent: true q-svc: true rabbit: true s-account: false s-container: false s-object: false s-proxy: false tempest: false tls-proxy: true extensions_to_txt: auto: true conf: true localrc: true log: true stackenv: true image_url: https://github.com/vexxhost/capo-image-elements/releases/latest/download/ubuntu-22.04-{{ kube_tag }}.qcow2 kube_tag: v1.34.7 network_driver: calico zuul: _inheritance_path: - '' - '' - '' - '' - '' - '' - '' - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: c9c08cb46e0e43b8920bb414aa26615d build_refs: - branch: main change: '1011' change_message: "fix(driver,sync,utils): converge stuck DELETE_IN_PROGRESS nodegroups under concurrent SSA race\n\n## Summary\n\nFour commits that together harden the magnum-cluster-api reconciler against the\nparallel `delete_nodegroup` race that wedges nodegroups in\n`DELETE_IN_PROGRESS` indefinitely while their `MachineDeployment` keeps\nrunning.\n\n| # | Commit | What it does |\n|---|---|---|\n| 1 | `fix(driver): guard reconciler against MachineDeploymentNotFound` | Stop the periodic `update_nodegroups_status` loop from being killed for the lifetime of the conductor pod when a concurrent topology mutation removes an MD between the GET and the spec-equality lookup. |\n| 2 | `fix(driver): recover stuck DELETE_IN_PROGRESS nodegroups with live MD` | Add a recovery branch in the reconciler: when an NG is `DELETE_IN_PROGRESS` but its MD is still present (a previous SSA was silently overwritten by a concurrent SSA with the same fieldManager + `force=True`), re-issue the topology removal so the delete eventually converges. |\n| 3 | `fix(sync): bump ClusterLock default TTL from 60s to 300s` | The 60s sherlock-KubernetesLock TTL with no heartbeat is shorter than the worst-case driver operation; on expiry, a second conductor can race the original. Bump to 5 minutes as an immediate mitigation; a proper fix needs a heartbeat / auto-renew loop. |\n| 4 | `fix(utils): strip resourceVersion from kube_apply_patch body` | pykube serialises the full GET'd object (which carries `metadata.resourceVersion`) into the SSA body. The apiserver then runs its generic optimistic-concurrency check and 409s on every parallel write \u2014 this is unrelated to SSA semantics. Strip RV/uid/generation from the body before applying. |\n\n## Race in brief\n\n1. Two `delete_nodegroup` calls run on different conductor pods.\n2. Both GET the Cluster CR, each removes a different `machineDeployments` entry, both Apply with the same `fieldManager` (`atmosphere-operator`) + `force=true`.\n3. Last writer wins: the earlier removal is silently restored.\n4. The DB row for the resurrected NG sits in `DELETE_IN_PROGRESS` forever \u2014 there was no recovery branch, and any subsequent `MachineDeploymentNotFound` raised by another NG would crash the whole reconcile loop.\n\n## Verification\n\nEnd-to-end on an AIO Atmosphere environment running this branch:\n\n- **Lock-protected real-driver `delete_nodegroup` \xD7 10 in parallel**: 4/5 trials clean (10/10 ok, 0 topology leftover); 1 trial transient lock-acquire timeout (lease churn from prior trials, not fix-related \u2014 recovery branch heals on next loop).\n- **Forced race (10 parallel SSA Applies bypassing ClusterLock)**: 5/5 trials reproduce the original bug (9/10 entries stuck). The recovery branch then converges them.\n- **Mixed parallel create + delete**: clean.\n- **TTL hold > 60s**: second `delete_nodegroup` waits the full duration, proving the 60\u2192300s bump works as intended.\n- **resourceVersion forced-race repro**: 10/10 \u2192 0/10 409s with the strip-RV fix.\n- **Unit tests**: 171/180 pass (9 pre-existing baseline failures unrelated to this change).\n- **Upgrade safety**: per-commit cross-version analysis + real `create_nodegroup` \u2192 `delete_nodegroup` E2E confirms the branch is upgrade-safe with mixed-ownership topology entries from older releases (the original `fieldManager` is preserved, no SSA ownership migration needed)." change_url: https://github.com/vexxhost/magnum-cluster-api/pull/1011 commit_id: 42045451f983ec59b3232712cac3ef3548d1ee87 patchset: 42045451f983ec59b3232712cac3ef3548d1ee87 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/magnum-cluster-api name: vexxhost/magnum-cluster-api short_name: magnum-cluster-api src_dir: src/github.com/vexxhost/magnum-cluster-api src_dir: src/github.com/vexxhost/magnum-cluster-api topic: null buildset: 21d5a9ede8574688ac6f45aa1ef2cdd8 buildset_refs: - branch: main change: '1011' change_message: "fix(driver,sync,utils): converge stuck DELETE_IN_PROGRESS nodegroups under concurrent SSA race\n\n## Summary\n\nFour commits that together harden the magnum-cluster-api reconciler against the\nparallel `delete_nodegroup` race that wedges nodegroups in\n`DELETE_IN_PROGRESS` indefinitely while their `MachineDeployment` keeps\nrunning.\n\n| # | Commit | What it does |\n|---|---|---|\n| 1 | `fix(driver): guard reconciler against MachineDeploymentNotFound` | Stop the periodic `update_nodegroups_status` loop from being killed for the lifetime of the conductor pod when a concurrent topology mutation removes an MD between the GET and the spec-equality lookup. |\n| 2 | `fix(driver): recover stuck DELETE_IN_PROGRESS nodegroups with live MD` | Add a recovery branch in the reconciler: when an NG is `DELETE_IN_PROGRESS` but its MD is still present (a previous SSA was silently overwritten by a concurrent SSA with the same fieldManager + `force=True`), re-issue the topology removal so the delete eventually converges. |\n| 3 | `fix(sync): bump ClusterLock default TTL from 60s to 300s` | The 60s sherlock-KubernetesLock TTL with no heartbeat is shorter than the worst-case driver operation; on expiry, a second conductor can race the original. Bump to 5 minutes as an immediate mitigation; a proper fix needs a heartbeat / auto-renew loop. |\n| 4 | `fix(utils): strip resourceVersion from kube_apply_patch body` | pykube serialises the full GET'd object (which carries `metadata.resourceVersion`) into the SSA body. The apiserver then runs its generic optimistic-concurrency check and 409s on every parallel write \u2014 this is unrelated to SSA semantics. Strip RV/uid/generation from the body before applying. |\n\n## Race in brief\n\n1. Two `delete_nodegroup` calls run on different conductor pods.\n2. Both GET the Cluster CR, each removes a different `machineDeployments` entry, both Apply with the same `fieldManager` (`atmosphere-operator`) + `force=true`.\n3. Last writer wins: the earlier removal is silently restored.\n4. The DB row for the resurrected NG sits in `DELETE_IN_PROGRESS` forever \u2014 there was no recovery branch, and any subsequent `MachineDeploymentNotFound` raised by another NG would crash the whole reconcile loop.\n\n## Verification\n\nEnd-to-end on an AIO Atmosphere environment running this branch:\n\n- **Lock-protected real-driver `delete_nodegroup` \xD7 10 in parallel**: 4/5 trials clean (10/10 ok, 0 topology leftover); 1 trial transient lock-acquire timeout (lease churn from prior trials, not fix-related \u2014 recovery branch heals on next loop).\n- **Forced race (10 parallel SSA Applies bypassing ClusterLock)**: 5/5 trials reproduce the original bug (9/10 entries stuck). The recovery branch then converges them.\n- **Mixed parallel create + delete**: clean.\n- **TTL hold > 60s**: second `delete_nodegroup` waits the full duration, proving the 60\u2192300s bump works as intended.\n- **resourceVersion forced-race repro**: 10/10 \u2192 0/10 409s with the strip-RV fix.\n- **Unit tests**: 171/180 pass (9 pre-existing baseline failures unrelated to this change).\n- **Upgrade safety**: per-commit cross-version analysis + real `create_nodegroup` \u2192 `delete_nodegroup` E2E confirms the branch is upgrade-safe with mixed-ownership topology entries from older releases (the original `fieldManager` is preserved, no SSA ownership migration needed)." change_url: https://github.com/vexxhost/magnum-cluster-api/pull/1011 commit_id: 42045451f983ec59b3232712cac3ef3548d1ee87 patchset: 42045451f983ec59b3232712cac3ef3548d1ee87 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/magnum-cluster-api name: vexxhost/magnum-cluster-api short_name: magnum-cluster-api src_dir: src/github.com/vexxhost/magnum-cluster-api src_dir: src/github.com/vexxhost/magnum-cluster-api topic: null change: '1011' change_message: "fix(driver,sync,utils): converge stuck DELETE_IN_PROGRESS nodegroups under concurrent SSA race\n\n## Summary\n\nFour commits that together harden the magnum-cluster-api reconciler against the\nparallel `delete_nodegroup` race that wedges nodegroups in\n`DELETE_IN_PROGRESS` indefinitely while their `MachineDeployment` keeps\nrunning.\n\n| # | Commit | What it does |\n|---|---|---|\n| 1 | `fix(driver): guard reconciler against MachineDeploymentNotFound` | Stop the periodic `update_nodegroups_status` loop from being killed for the lifetime of the conductor pod when a concurrent topology mutation removes an MD between the GET and the spec-equality lookup. |\n| 2 | `fix(driver): recover stuck DELETE_IN_PROGRESS nodegroups with live MD` | Add a recovery branch in the reconciler: when an NG is `DELETE_IN_PROGRESS` but its MD is still present (a previous SSA was silently overwritten by a concurrent SSA with the same fieldManager + `force=True`), re-issue the topology removal so the delete eventually converges. |\n| 3 | `fix(sync): bump ClusterLock default TTL from 60s to 300s` | The 60s sherlock-KubernetesLock TTL with no heartbeat is shorter than the worst-case driver operation; on expiry, a second conductor can race the original. Bump to 5 minutes as an immediate mitigation; a proper fix needs a heartbeat / auto-renew loop. |\n| 4 | `fix(utils): strip resourceVersion from kube_apply_patch body` | pykube serialises the full GET'd object (which carries `metadata.resourceVersion`) into the SSA body. The apiserver then runs its generic optimistic-concurrency check and 409s on every parallel write \u2014 this is unrelated to SSA semantics. Strip RV/uid/generation from the body before applying. |\n\n## Race in brief\n\n1. Two `delete_nodegroup` calls run on different conductor pods.\n2. Both GET the Cluster CR, each removes a different `machineDeployments` entry, both Apply with the same `fieldManager` (`atmosphere-operator`) + `force=true`.\n3. Last writer wins: the earlier removal is silently restored.\n4. The DB row for the resurrected NG sits in `DELETE_IN_PROGRESS` forever \u2014 there was no recovery branch, and any subsequent `MachineDeploymentNotFound` raised by another NG would crash the whole reconcile loop.\n\n## Verification\n\nEnd-to-end on an AIO Atmosphere environment running this branch:\n\n- **Lock-protected real-driver `delete_nodegroup` \xD7 10 in parallel**: 4/5 trials clean (10/10 ok, 0 topology leftover); 1 trial transient lock-acquire timeout (lease churn from prior trials, not fix-related \u2014 recovery branch heals on next loop).\n- **Forced race (10 parallel SSA Applies bypassing ClusterLock)**: 5/5 trials reproduce the original bug (9/10 entries stuck). The recovery branch then converges them.\n- **Mixed parallel create + delete**: clean.\n- **TTL hold > 60s**: second `delete_nodegroup` waits the full duration, proving the 60\u2192300s bump works as intended.\n- **resourceVersion forced-race repro**: 10/10 \u2192 0/10 409s with the strip-RV fix.\n- **Unit tests**: 171/180 pass (9 pre-existing baseline failures unrelated to this change).\n- **Upgrade safety**: per-commit cross-version analysis + real `create_nodegroup` \u2192 `delete_nodegroup` E2E confirms the branch is upgrade-safe with mixed-ownership topology entries from older releases (the original `fieldManager` is preserved, no SSA ownership migration needed)." change_url: https://github.com/vexxhost/magnum-cluster-api/pull/1011 child_jobs: [] commit_id: 42045451f983ec59b3232712cac3ef3548d1ee87 event_id: 4127c830-47cd-11f1-8e2f-d6d90206a647 executor: hostname: 2d72f0692154 inventory_file: /var/lib/zuul/builds/c9c08cb46e0e43b8920bb414aa26615d/ansible/inventory.yaml log_root: /var/lib/zuul/builds/c9c08cb46e0e43b8920bb414aa26615d/work/logs result_data_file: /var/lib/zuul/builds/c9c08cb46e0e43b8920bb414aa26615d/work/results.json src_root: /var/lib/zuul/builds/c9c08cb46e0e43b8920bb414aa26615d/work/src work_root: /var/lib/zuul/builds/c9c08cb46e0e43b8920bb414aa26615d/work include_vars: [] items: - branch: main change: '1011' change_message: "fix(driver,sync,utils): converge stuck DELETE_IN_PROGRESS nodegroups under concurrent SSA race\n\n## Summary\n\nFour commits that together harden the magnum-cluster-api reconciler against the\nparallel `delete_nodegroup` race that wedges nodegroups in\n`DELETE_IN_PROGRESS` indefinitely while their `MachineDeployment` keeps\nrunning.\n\n| # | Commit | What it does |\n|---|---|---|\n| 1 | `fix(driver): guard reconciler against MachineDeploymentNotFound` | Stop the periodic `update_nodegroups_status` loop from being killed for the lifetime of the conductor pod when a concurrent topology mutation removes an MD between the GET and the spec-equality lookup. |\n| 2 | `fix(driver): recover stuck DELETE_IN_PROGRESS nodegroups with live MD` | Add a recovery branch in the reconciler: when an NG is `DELETE_IN_PROGRESS` but its MD is still present (a previous SSA was silently overwritten by a concurrent SSA with the same fieldManager + `force=True`), re-issue the topology removal so the delete eventually converges. |\n| 3 | `fix(sync): bump ClusterLock default TTL from 60s to 300s` | The 60s sherlock-KubernetesLock TTL with no heartbeat is shorter than the worst-case driver operation; on expiry, a second conductor can race the original. Bump to 5 minutes as an immediate mitigation; a proper fix needs a heartbeat / auto-renew loop. |\n| 4 | `fix(utils): strip resourceVersion from kube_apply_patch body` | pykube serialises the full GET'd object (which carries `metadata.resourceVersion`) into the SSA body. The apiserver then runs its generic optimistic-concurrency check and 409s on every parallel write \u2014 this is unrelated to SSA semantics. Strip RV/uid/generation from the body before applying. |\n\n## Race in brief\n\n1. Two `delete_nodegroup` calls run on different conductor pods.\n2. Both GET the Cluster CR, each removes a different `machineDeployments` entry, both Apply with the same `fieldManager` (`atmosphere-operator`) + `force=true`.\n3. Last writer wins: the earlier removal is silently restored.\n4. The DB row for the resurrected NG sits in `DELETE_IN_PROGRESS` forever \u2014 there was no recovery branch, and any subsequent `MachineDeploymentNotFound` raised by another NG would crash the whole reconcile loop.\n\n## Verification\n\nEnd-to-end on an AIO Atmosphere environment running this branch:\n\n- **Lock-protected real-driver `delete_nodegroup` \xD7 10 in parallel**: 4/5 trials clean (10/10 ok, 0 topology leftover); 1 trial transient lock-acquire timeout (lease churn from prior trials, not fix-related \u2014 recovery branch heals on next loop).\n- **Forced race (10 parallel SSA Applies bypassing ClusterLock)**: 5/5 trials reproduce the original bug (9/10 entries stuck). The recovery branch then converges them.\n- **Mixed parallel create + delete**: clean.\n- **TTL hold > 60s**: second `delete_nodegroup` waits the full duration, proving the 60\u2192300s bump works as intended.\n- **resourceVersion forced-race repro**: 10/10 \u2192 0/10 409s with the strip-RV fix.\n- **Unit tests**: 171/180 pass (9 pre-existing baseline failures unrelated to this change).\n- **Upgrade safety**: per-commit cross-version analysis + real `create_nodegroup` \u2192 `delete_nodegroup` E2E confirms the branch is upgrade-safe with mixed-ownership topology entries from older releases (the original `fieldManager` is preserved, no SSA ownership migration needed)." change_url: https://github.com/vexxhost/magnum-cluster-api/pull/1011 commit_id: 42045451f983ec59b3232712cac3ef3548d1ee87 patchset: 42045451f983ec59b3232712cac3ef3548d1ee87 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/magnum-cluster-api name: vexxhost/magnum-cluster-api short_name: magnum-cluster-api src_dir: src/github.com/vexxhost/magnum-cluster-api topic: null job: magnum-cluster-api-hydrophone-v1.34.7-calico jobtags: [] max_attempts: 3 message: Zml4KGRyaXZlcixzeW5jLHV0aWxzKTogY29udmVyZ2Ugc3R1Y2sgREVMRVRFX0lOX1BST0dSRVNTIG5vZGVncm91cHMgdW5kZXIgY29uY3VycmVudCBTU0EgcmFjZQoKIyMgU3VtbWFyeQoKRm91ciBjb21taXRzIHRoYXQgdG9nZXRoZXIgaGFyZGVuIHRoZSBtYWdudW0tY2x1c3Rlci1hcGkgcmVjb25jaWxlciBhZ2FpbnN0IHRoZQpwYXJhbGxlbCBgZGVsZXRlX25vZGVncm91cGAgcmFjZSB0aGF0IHdlZGdlcyBub2RlZ3JvdXBzIGluCmBERUxFVEVfSU5fUFJPR1JFU1NgIGluZGVmaW5pdGVseSB3aGlsZSB0aGVpciBgTWFjaGluZURlcGxveW1lbnRgIGtlZXBzCnJ1bm5pbmcuCgp8ICMgfCBDb21taXQgfCBXaGF0IGl0IGRvZXMgfAp8LS0tfC0tLXwtLS18CnwgMSB8IGBmaXgoZHJpdmVyKTogZ3VhcmQgcmVjb25jaWxlciBhZ2FpbnN0IE1hY2hpbmVEZXBsb3ltZW50Tm90Rm91bmRgIHwgU3RvcCB0aGUgcGVyaW9kaWMgYHVwZGF0ZV9ub2RlZ3JvdXBzX3N0YXR1c2AgbG9vcCBmcm9tIGJlaW5nIGtpbGxlZCBmb3IgdGhlIGxpZmV0aW1lIG9mIHRoZSBjb25kdWN0b3IgcG9kIHdoZW4gYSBjb25jdXJyZW50IHRvcG9sb2d5IG11dGF0aW9uIHJlbW92ZXMgYW4gTUQgYmV0d2VlbiB0aGUgR0VUIGFuZCB0aGUgc3BlYy1lcXVhbGl0eSBsb29rdXAuIHwKfCAyIHwgYGZpeChkcml2ZXIpOiByZWNvdmVyIHN0dWNrIERFTEVURV9JTl9QUk9HUkVTUyBub2RlZ3JvdXBzIHdpdGggbGl2ZSBNRGAgfCBBZGQgYSByZWNvdmVyeSBicmFuY2ggaW4gdGhlIHJlY29uY2lsZXI6IHdoZW4gYW4gTkcgaXMgYERFTEVURV9JTl9QUk9HUkVTU2AgYnV0IGl0cyBNRCBpcyBzdGlsbCBwcmVzZW50IChhIHByZXZpb3VzIFNTQSB3YXMgc2lsZW50bHkgb3ZlcndyaXR0ZW4gYnkgYSBjb25jdXJyZW50IFNTQSB3aXRoIHRoZSBzYW1lIGZpZWxkTWFuYWdlciArIGBmb3JjZT1UcnVlYCksIHJlLWlzc3VlIHRoZSB0b3BvbG9neSByZW1vdmFsIHNvIHRoZSBkZWxldGUgZXZlbnR1YWxseSBjb252ZXJnZXMuIHwKfCAzIHwgYGZpeChzeW5jKTogYnVtcCBDbHVzdGVyTG9jayBkZWZhdWx0IFRUTCBmcm9tIDYwcyB0byAzMDBzYCB8IFRoZSA2MHMgc2hlcmxvY2stS3ViZXJuZXRlc0xvY2sgVFRMIHdpdGggbm8gaGVhcnRiZWF0IGlzIHNob3J0ZXIgdGhhbiB0aGUgd29yc3QtY2FzZSBkcml2ZXIgb3BlcmF0aW9uOyBvbiBleHBpcnksIGEgc2Vjb25kIGNvbmR1Y3RvciBjYW4gcmFjZSB0aGUgb3JpZ2luYWwuIEJ1bXAgdG8gNSBtaW51dGVzIGFzIGFuIGltbWVkaWF0ZSBtaXRpZ2F0aW9uOyBhIHByb3BlciBmaXggbmVlZHMgYSBoZWFydGJlYXQgLyBhdXRvLXJlbmV3IGxvb3AuIHwKfCA0IHwgYGZpeCh1dGlscyk6IHN0cmlwIHJlc291cmNlVmVyc2lvbiBmcm9tIGt1YmVfYXBwbHlfcGF0Y2ggYm9keWAgfCBweWt1YmUgc2VyaWFsaXNlcyB0aGUgZnVsbCBHRVQnZCBvYmplY3QgKHdoaWNoIGNhcnJpZXMgYG1ldGFkYXRhLnJlc291cmNlVmVyc2lvbmApIGludG8gdGhlIFNTQSBib2R5LiBUaGUgYXBpc2VydmVyIHRoZW4gcnVucyBpdHMgZ2VuZXJpYyBvcHRpbWlzdGljLWNvbmN1cnJlbmN5IGNoZWNrIGFuZCA0MDlzIG9uIGV2ZXJ5IHBhcmFsbGVsIHdyaXRlIOKAlCB0aGlzIGlzIHVucmVsYXRlZCB0byBTU0Egc2VtYW50aWNzLiBTdHJpcCBSVi91aWQvZ2VuZXJhdGlvbiBmcm9tIHRoZSBib2R5IGJlZm9yZSBhcHBseWluZy4gfAoKIyMgUmFjZSBpbiBicmllZgoKMS4gVHdvIGBkZWxldGVfbm9kZWdyb3VwYCBjYWxscyBydW4gb24gZGlmZmVyZW50IGNvbmR1Y3RvciBwb2RzLgoyLiBCb3RoIEdFVCB0aGUgQ2x1c3RlciBDUiwgZWFjaCByZW1vdmVzIGEgZGlmZmVyZW50IGBtYWNoaW5lRGVwbG95bWVudHNgIGVudHJ5LCBib3RoIEFwcGx5IHdpdGggdGhlIHNhbWUgYGZpZWxkTWFuYWdlcmAgKGBhdG1vc3BoZXJlLW9wZXJhdG9yYCkgKyBgZm9yY2U9dHJ1ZWAuCjMuIExhc3Qgd3JpdGVyIHdpbnM6IHRoZSBlYXJsaWVyIHJlbW92YWwgaXMgc2lsZW50bHkgcmVzdG9yZWQuCjQuIFRoZSBEQiByb3cgZm9yIHRoZSByZXN1cnJlY3RlZCBORyBzaXRzIGluIGBERUxFVEVfSU5fUFJPR1JFU1NgIGZvcmV2ZXIg4oCUIHRoZXJlIHdhcyBubyByZWNvdmVyeSBicmFuY2gsIGFuZCBhbnkgc3Vic2VxdWVudCBgTWFjaGluZURlcGxveW1lbnROb3RGb3VuZGAgcmFpc2VkIGJ5IGFub3RoZXIgTkcgd291bGQgY3Jhc2ggdGhlIHdob2xlIHJlY29uY2lsZSBsb29wLgoKIyMgVmVyaWZpY2F0aW9uCgpFbmQtdG8tZW5kIG9uIGFuIEFJTyBBdG1vc3BoZXJlIGVudmlyb25tZW50IHJ1bm5pbmcgdGhpcyBicmFuY2g6CgotICoqTG9jay1wcm90ZWN0ZWQgcmVhbC1kcml2ZXIgYGRlbGV0ZV9ub2RlZ3JvdXBgIMOXIDEwIGluIHBhcmFsbGVsKio6IDQvNSB0cmlhbHMgY2xlYW4gKDEwLzEwIG9rLCAwIHRvcG9sb2d5IGxlZnRvdmVyKTsgMSB0cmlhbCB0cmFuc2llbnQgbG9jay1hY3F1aXJlIHRpbWVvdXQgKGxlYXNlIGNodXJuIGZyb20gcHJpb3IgdHJpYWxzLCBub3QgZml4LXJlbGF0ZWQg4oCUIHJlY292ZXJ5IGJyYW5jaCBoZWFscyBvbiBuZXh0IGxvb3ApLgotICoqRm9yY2VkIHJhY2UgKDEwIHBhcmFsbGVsIFNTQSBBcHBsaWVzIGJ5cGFzc2luZyBDbHVzdGVyTG9jaykqKjogNS81IHRyaWFscyByZXByb2R1Y2UgdGhlIG9yaWdpbmFsIGJ1ZyAoOS8xMCBlbnRyaWVzIHN0dWNrKS4gVGhlIHJlY292ZXJ5IGJyYW5jaCB0aGVuIGNvbnZlcmdlcyB0aGVtLgotICoqTWl4ZWQgcGFyYWxsZWwgY3JlYXRlICsgZGVsZXRlKio6IGNsZWFuLgotICoqVFRMIGhvbGQgPiA2MHMqKjogc2Vjb25kIGBkZWxldGVfbm9kZWdyb3VwYCB3YWl0cyB0aGUgZnVsbCBkdXJhdGlvbiwgcHJvdmluZyB0aGUgNjDihpIzMDBzIGJ1bXAgd29ya3MgYXMgaW50ZW5kZWQuCi0gKipyZXNvdXJjZVZlcnNpb24gZm9yY2VkLXJhY2UgcmVwcm8qKjogMTAvMTAg4oaSIDAvMTAgNDA5cyB3aXRoIHRoZSBzdHJpcC1SViBmaXguCi0gKipVbml0IHRlc3RzKio6IDE3MS8xODAgcGFzcyAoOSBwcmUtZXhpc3RpbmcgYmFzZWxpbmUgZmFpbHVyZXMgdW5yZWxhdGVkIHRvIHRoaXMgY2hhbmdlKS4KLSAqKlVwZ3JhZGUgc2FmZXR5Kio6IHBlci1jb21taXQgY3Jvc3MtdmVyc2lvbiBhbmFseXNpcyArIHJlYWwgYGNyZWF0ZV9ub2RlZ3JvdXBgIOKGkiBgZGVsZXRlX25vZGVncm91cGAgRTJFIGNvbmZpcm1zIHRoZSBicmFuY2ggaXMgdXBncmFkZS1zYWZlIHdpdGggbWl4ZWQtb3duZXJzaGlwIHRvcG9sb2d5IGVudHJpZXMgZnJvbSBvbGRlciByZWxlYXNlcyAodGhlIG9yaWdpbmFsIGBmaWVsZE1hbmFnZXJgIGlzIHByZXNlcnZlZCwgbm8gU1NBIG93bmVyc2hpcCBtaWdyYXRpb24gbmVlZGVkKS4= override_checkout: master patchset: 42045451f983ec59b3232712cac3ef3548d1ee87 pipeline: check playbook_context: playbook_projects: trusted/project_0/github.com/vexxhost/zuul-config: canonical_name: github.com/vexxhost/zuul-config checkout: main commit: 298983cd1253e6833abdb49d87d912527e0e6597 trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 79fe3eb1d01f8ac5739b0b7bc4759c407b6e248d trusted/project_2/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: a6e68243e02ef030ce5e75f8b67630880c475f33 untrusted/project_0/opendev.org/openstack/devstack: canonical_name: opendev.org/openstack/devstack checkout: master commit: 03ece8f88040be9b0b14dd1cfe93076ad2419a80 untrusted/project_1/opendev.org/openstack/openstack-zuul-jobs: canonical_name: opendev.org/openstack/openstack-zuul-jobs checkout: master commit: 74b9cec5beecc1fc5bc89251ac9de475824e2cbb untrusted/project_2/github.com/vexxhost/zuul-config: canonical_name: github.com/vexxhost/zuul-config checkout: main commit: 298983cd1253e6833abdb49d87d912527e0e6597 untrusted/project_3/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 79fe3eb1d01f8ac5739b0b7bc4759c407b6e248d untrusted/project_4/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: a6e68243e02ef030ce5e75f8b67630880c475f33 untrusted/project_5/github.com/vexxhost/magnum-cluster-api: canonical_name: github.com/vexxhost/magnum-cluster-api checkout: main commit: 42045451f983ec59b3232712cac3ef3548d1ee87 playbooks: - path: untrusted/project_5/github.com/vexxhost/magnum-cluster-api/zuul.d/playbooks/hydrophone/run.yml roles: - checkout: master checkout_description: job override ref link_name: ansible/playbook_0/role_1/devstack link_target: untrusted/project_0/opendev.org/openstack/devstack role_path: ansible/playbook_0/role_1/devstack/roles - checkout: master checkout_description: job override ref link_name: ansible/playbook_0/role_2/openstack-zuul-jobs link_target: untrusted/project_1/opendev.org/openstack/openstack-zuul-jobs role_path: ansible/playbook_0/role_2/openstack-zuul-jobs/roles - checkout: master checkout_description: job override ref link_name: ansible/playbook_0/role_4/zuul-jobs link_target: untrusted/project_3/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_4/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/playbook_0/role_5/zuul-jobs link_target: untrusted/project_4/github.com/vexxhost/zuul-jobs role_path: ansible/playbook_0/role_5/zuul-jobs/roles post_playbooks: - path: untrusted/project_5/github.com/vexxhost/magnum-cluster-api/zuul.d/playbooks/hydrophone/post.yml roles: - checkout: master checkout_description: job override ref link_name: ansible/post_playbook_0/role_1/devstack link_target: untrusted/project_0/opendev.org/openstack/devstack role_path: ansible/post_playbook_0/role_1/devstack/roles - checkout: master checkout_description: job override ref link_name: ansible/post_playbook_0/role_2/openstack-zuul-jobs link_target: untrusted/project_1/opendev.org/openstack/openstack-zuul-jobs role_path: ansible/post_playbook_0/role_2/openstack-zuul-jobs/roles - checkout: master checkout_description: job override ref link_name: ansible/post_playbook_0/role_4/zuul-jobs link_target: untrusted/project_3/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_0/role_4/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_0/role_5/zuul-jobs link_target: untrusted/project_4/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_0/role_5/zuul-jobs/roles - path: untrusted/project_0/opendev.org/openstack/devstack/playbooks/post.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_1/role_0/devstack link_target: untrusted/project_0/opendev.org/openstack/devstack role_path: ansible/post_playbook_1/role_0/devstack/roles - checkout: master checkout_description: job override ref link_name: ansible/post_playbook_1/role_1/openstack-zuul-jobs link_target: untrusted/project_1/opendev.org/openstack/openstack-zuul-jobs role_path: ansible/post_playbook_1/role_1/openstack-zuul-jobs/roles - checkout: master checkout_description: job override ref link_name: ansible/post_playbook_1/role_3/zuul-jobs link_target: untrusted/project_3/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_1/role_3/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_1/role_4/zuul-jobs link_target: untrusted/project_4/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_1/role_4/zuul-jobs/roles - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/post.yaml roles: - checkout: master checkout_description: job override ref link_name: ansible/post_playbook_2/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_2/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_2/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_2/role_2/zuul-jobs/roles - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/post-logs.yaml roles: - checkout: master checkout_description: job override ref link_name: ansible/post_playbook_3/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_3/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_3/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_3/role_2/zuul-jobs/roles pre_playbooks: - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/pre.yaml roles: - checkout: master checkout_description: job override ref link_name: ansible/pre_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_0/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/pre_playbook_0/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/pre_playbook_0/role_2/zuul-jobs/roles - path: untrusted/project_0/opendev.org/openstack/devstack/playbooks/pre.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_1/role_0/devstack link_target: untrusted/project_0/opendev.org/openstack/devstack role_path: ansible/pre_playbook_1/role_0/devstack/roles - checkout: master checkout_description: job override ref link_name: ansible/pre_playbook_1/role_1/openstack-zuul-jobs link_target: untrusted/project_1/opendev.org/openstack/openstack-zuul-jobs role_path: ansible/pre_playbook_1/role_1/openstack-zuul-jobs/roles - checkout: master checkout_description: job override ref link_name: ansible/pre_playbook_1/role_3/zuul-jobs link_target: untrusted/project_3/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_1/role_3/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/pre_playbook_1/role_4/zuul-jobs link_target: untrusted/project_4/github.com/vexxhost/zuul-jobs role_path: ansible/pre_playbook_1/role_4/zuul-jobs/roles - path: untrusted/project_5/github.com/vexxhost/magnum-cluster-api/zuul.d/playbooks/hydrophone/pre.yml roles: - checkout: master checkout_description: job override ref link_name: ansible/pre_playbook_2/role_1/devstack link_target: untrusted/project_0/opendev.org/openstack/devstack role_path: ansible/pre_playbook_2/role_1/devstack/roles - checkout: master checkout_description: job override ref link_name: ansible/pre_playbook_2/role_2/openstack-zuul-jobs link_target: untrusted/project_1/opendev.org/openstack/openstack-zuul-jobs role_path: ansible/pre_playbook_2/role_2/openstack-zuul-jobs/roles - checkout: master checkout_description: job override ref link_name: ansible/pre_playbook_2/role_4/zuul-jobs link_target: untrusted/project_3/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_2/role_4/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/pre_playbook_2/role_5/zuul-jobs link_target: untrusted/project_4/github.com/vexxhost/zuul-jobs role_path: ansible/pre_playbook_2/role_5/zuul-jobs/roles post_review: false post_timeout: null pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/vexxhost/magnum-cluster-api name: vexxhost/magnum-cluster-api short_name: magnum-cluster-api src_dir: src/github.com/vexxhost/magnum-cluster-api projects: github.com/novnc/novnc: canonical_hostname: github.com canonical_name: github.com/novnc/novnc checkout: master checkout_description: job override ref commit: 8e1ebdffba02e651c399dacef841f8941f6ad6e4 name: novnc/novnc required: true short_name: novnc src_dir: src/github.com/novnc/novnc github.com/vexxhost/magnum-cluster-api: canonical_hostname: github.com canonical_name: github.com/vexxhost/magnum-cluster-api checkout: main checkout_description: zuul branch commit: 42045451f983ec59b3232712cac3ef3548d1ee87 name: vexxhost/magnum-cluster-api required: false short_name: magnum-cluster-api src_dir: src/github.com/vexxhost/magnum-cluster-api opendev.org/openstack/barbican: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/barbican checkout: master checkout_description: job override ref commit: f47012f3e19d32fa42fb2df9dd3556819af9680b name: openstack/barbican required: true short_name: barbican src_dir: src/opendev.org/openstack/barbican opendev.org/openstack/cinder: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/cinder checkout: master checkout_description: job override ref commit: 65bba6830d578d7fe3138dfa65228dfda19eb49e name: openstack/cinder required: true short_name: cinder src_dir: src/opendev.org/openstack/cinder opendev.org/openstack/devstack: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/devstack checkout: master checkout_description: job override ref commit: 03ece8f88040be9b0b14dd1cfe93076ad2419a80 name: openstack/devstack required: true short_name: devstack src_dir: src/opendev.org/openstack/devstack opendev.org/openstack/glance: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/glance checkout: master checkout_description: job override ref commit: ad10cfd88ea416ffeaea67daf21d03242f9befb5 name: openstack/glance required: true short_name: glance src_dir: src/opendev.org/openstack/glance opendev.org/openstack/keystone: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/keystone checkout: master checkout_description: job override ref commit: 2230026f77a8ed50493d2d58be9120910ceb2089 name: openstack/keystone required: true short_name: keystone src_dir: src/opendev.org/openstack/keystone opendev.org/openstack/magnum: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/magnum checkout: master checkout_description: job override ref commit: cdb19fce44b3e6d9e07b2e5b3e1b2c40c5de870c name: openstack/magnum required: true short_name: magnum src_dir: src/opendev.org/openstack/magnum opendev.org/openstack/manila: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/manila checkout: master checkout_description: job override ref commit: db4b12fab95a142f574b63576917959ea95e3ce0 name: openstack/manila required: true short_name: manila src_dir: src/opendev.org/openstack/manila opendev.org/openstack/neutron: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/neutron checkout: master checkout_description: job override ref commit: a52c613be8132e7092ec3826444f08809fcec818 name: openstack/neutron required: true short_name: neutron src_dir: src/opendev.org/openstack/neutron opendev.org/openstack/nova: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/nova checkout: master checkout_description: job override ref commit: 2e0bfb1102d8af376c110a169eba42e0905dae07 name: openstack/nova required: true short_name: nova src_dir: src/opendev.org/openstack/nova opendev.org/openstack/octavia: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/octavia checkout: master checkout_description: job override ref commit: 19dec79e4b223d1cfca18fde88c01307405c0b34 name: openstack/octavia required: true short_name: octavia src_dir: src/opendev.org/openstack/octavia opendev.org/openstack/os-test-images: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/os-test-images checkout: master checkout_description: job override ref commit: 5d0367e03788764f41da8effffa14e3eac513201 name: openstack/os-test-images required: true short_name: os-test-images src_dir: src/opendev.org/openstack/os-test-images opendev.org/openstack/ovn-octavia-provider: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/ovn-octavia-provider checkout: master checkout_description: job override ref commit: a04e63332c1a1fb2f1aceb20892f9aabefb75cf7 name: openstack/ovn-octavia-provider required: true short_name: ovn-octavia-provider src_dir: src/opendev.org/openstack/ovn-octavia-provider opendev.org/openstack/placement: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/placement checkout: master checkout_description: job override ref commit: e3db398fba279721121892323e6260c6932797c1 name: openstack/placement required: true short_name: placement src_dir: src/opendev.org/openstack/placement opendev.org/openstack/python-magnumclient: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/python-magnumclient checkout: master checkout_description: job override ref commit: bfc9dbc2aa9a113c12e591a87f774a6d986a981f name: openstack/python-magnumclient required: true short_name: python-magnumclient src_dir: src/opendev.org/openstack/python-magnumclient opendev.org/openstack/requirements: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/requirements checkout: master checkout_description: job override ref commit: ed679515f6e053d20b05ffcc30b3172de864f7f1 name: openstack/requirements required: true short_name: requirements src_dir: src/opendev.org/openstack/requirements opendev.org/openstack/swift: canonical_hostname: opendev.org canonical_name: opendev.org/openstack/swift checkout: master checkout_description: job override ref commit: 453e2356ba92fdefbe9e0886028d8414a1063684 name: openstack/swift required: true short_name: swift src_dir: src/opendev.org/openstack/swift ref: refs/pull/1011/head resources: {} tenant: oss timeout: 7200 topic: null voting: true zuul_copy_output: /etc/ceph: logs /etc/glusterfs/glusterd.vol: logs /etc/libvirt: logs /etc/lvm: logs /etc/resolv.conf: logs /etc/sudoers: logs /etc/sudoers.d: logs /var/log/ceph: logs /var/log/glusterfs: logs /var/log/libvirt: logs /var/log/mysql: logs /var/log/openvswitch: logs /var/log/postgresql: logs /var/log/rabbitmq: logs /var/log/unbound.log: logs '{{ devstack_conf_dir }}/.localrc.auto': logs '{{ devstack_conf_dir }}/.stackenv': logs '{{ devstack_conf_dir }}/local.conf': logs '{{ devstack_conf_dir }}/localrc': logs '{{ devstack_full_log}}': logs '{{ devstack_log_dir }}/atop': logs '{{ devstack_log_dir }}/devstacklog.txt': logs '{{ devstack_log_dir }}/devstacklog.txt.summary': logs '{{ devstack_log_dir }}/dstat-csv.log': logs '{{ devstack_log_dir }}/qemu.coredump': logs '{{ devstack_log_dir }}/tcpdump.pcap': logs '{{ devstack_log_dir }}/worlddump-latest.txt': logs '{{ stage_dir }}/apache': logs '{{ stage_dir }}/apache_config': logs '{{ stage_dir }}/audit.log': logs '{{ stage_dir }}/core': logs '{{ stage_dir }}/deprecations.log': logs '{{ stage_dir }}/df.txt': logs '{{ stage_dir }}/dpkg-l.txt': logs '{{ stage_dir }}/etc': logs '{{ stage_dir }}/iptables.txt': logs '{{ stage_dir }}/listen53.txt': logs '{{ stage_dir }}/mount.txt': logs '{{ stage_dir }}/performance.json': logs '{{ stage_dir }}/pip2-freeze.txt': logs '{{ stage_dir }}/pip3-freeze.txt': logs '{{ stage_dir }}/rpm-qa.txt': logs '{{ stage_dir }}/services.txt': logs '{{ stage_dir }}/verify_tempest_conf.log': logs