all: children: zuul_unreachable: hosts: {} hosts: ubuntu-noble: ansible_connection: ssh ansible_host: 162.253.55.5 ansible_port: 22 ansible_python_interpreter: auto ansible_user: zuul nodepool: az: yul2 cloud: public external_id: 1149e5cb-4ccc-4e6a-b311-c74642e650c3 host_id: a01cb4e3d8698a10b6af7a23bf700f5af12859fba177738e8fa3822e interface_ip: 162.253.55.5 label: ubuntu-noble node_properties: {} private_ipv4: 162.253.55.5 private_ipv6: null provider: yul1 public_ipv4: 162.253.55.5 public_ipv6: 2604:e100:1:0:f816:3eff:fe3b:5163 region: ca-ymq-1 slot: null zuul_node: az: yul2 cloud: public external_id: 1149e5cb-4ccc-4e6a-b311-c74642e650c3 host_id: a01cb4e3d8698a10b6af7a23bf700f5af12859fba177738e8fa3822e interface_ip: 162.253.55.5 label: ubuntu-noble node_properties: {} private_ipv4: 162.253.55.5 private_ipv6: null provider: yul1 public_ipv4: 162.253.55.5 public_ipv6: 2604:e100:1:0:f816:3eff:fe3b:5163 region: ca-ymq-1 slot: null uuid: null vars: zuul: _inheritance_path: - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: 2e00dc2c478b41acb073b1087cb305dc build_refs: - branch: main change: '324' change_message: "feat(kubernetes): upgrade control plane nodes one at a time with health checks\n\nBottom of the upgrade stack (#323 merged): this PR targets `main`, and the rest of the stack builds on it. Replaces #314, which GitHub closed as merged by mistake when the stack was reordered.\n\nDuring a control plane upgrade, two etcd members can be down at once. After `kubeadm upgrade apply`, the role runs `kubeadm upgrade node` on the other control plane nodes with `throttle: 1`, which only waits for the previous node's command to return. kubeadm returns once it has replaced the node's static pod manifests, while etcd, the API server, the controller manager and the scheduler are still restarting. The next node can then start restarting its own. On a three-node control plane that's two of three etcd members down together, which loses quorum.\n\nThis PR upgrades the control plane nodes in a loop (`control-plane-upgrade-node.yml`). After `kubeadm upgrade apply`, and after each node that needed an upgrade, it waits (`health-check.yml`) for:\n\n- the API's `/readyz` endpoint;\n- one `etcd`, `kube-apiserver`, `kube-controller-manager` and `kube-scheduler` pod per control plane node, all Ready;\n- the CoreDNS rollout.\n\nNew defaults: `kubernetes_health_check_components`, `kubernetes_health_check_retries` (30), `kubernetes_health_check_delay` (10) and `kubernetes_health_check_timeout` (`300s`).\n\n**CI change:** the molecule jobs now set `kubernetes_coredns_node_selector: {node-role.kubernetes.io/control-plane: \"\"}`. The default selector, `openstack-control-plane: enabled`, is an Atmosphere node label that CI nodes don't have. CoreDNS was never schedulable in CI, and the new \"Wait for CoreDNS to be available\" check (correctly) timed out because no CoreDNS pod was running during the upgrade. This also lets CI exercise #323's CoreDNS patch.\n\nValidation: pre-commit (including ansible-lint) passes. The three-node `cluster-upgrade` Zuul job (1.33.4 \u2192 1.34.0) runs this path.\n\nStacked on #323: this PR targets that PR's branch, so the diff shows only this change. It moves to `main` once #323 merges.\n\n\n\n" change_url: https://github.com/vexxhost/ansible-collection-kubernetes/pull/324 commit_id: 41423f1a9031c1d89ff7894e0318dfbaded337e6 patchset: 41423f1a9031c1d89ff7894e0318dfbaded337e6 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/ansible-collection-kubernetes name: vexxhost/ansible-collection-kubernetes short_name: ansible-collection-kubernetes src_dir: src/github.com/vexxhost/ansible-collection-kubernetes src_dir: src/github.com/vexxhost/ansible-collection-kubernetes topic: null buildset: 5ef95a8f328a45229d94cec993993834 buildset_refs: - branch: main change: '324' change_message: "feat(kubernetes): upgrade control plane nodes one at a time with health checks\n\nBottom of the upgrade stack (#323 merged): this PR targets `main`, and the rest of the stack builds on it. Replaces #314, which GitHub closed as merged by mistake when the stack was reordered.\n\nDuring a control plane upgrade, two etcd members can be down at once. After `kubeadm upgrade apply`, the role runs `kubeadm upgrade node` on the other control plane nodes with `throttle: 1`, which only waits for the previous node's command to return. kubeadm returns once it has replaced the node's static pod manifests, while etcd, the API server, the controller manager and the scheduler are still restarting. The next node can then start restarting its own. On a three-node control plane that's two of three etcd members down together, which loses quorum.\n\nThis PR upgrades the control plane nodes in a loop (`control-plane-upgrade-node.yml`). After `kubeadm upgrade apply`, and after each node that needed an upgrade, it waits (`health-check.yml`) for:\n\n- the API's `/readyz` endpoint;\n- one `etcd`, `kube-apiserver`, `kube-controller-manager` and `kube-scheduler` pod per control plane node, all Ready;\n- the CoreDNS rollout.\n\nNew defaults: `kubernetes_health_check_components`, `kubernetes_health_check_retries` (30), `kubernetes_health_check_delay` (10) and `kubernetes_health_check_timeout` (`300s`).\n\n**CI change:** the molecule jobs now set `kubernetes_coredns_node_selector: {node-role.kubernetes.io/control-plane: \"\"}`. The default selector, `openstack-control-plane: enabled`, is an Atmosphere node label that CI nodes don't have. CoreDNS was never schedulable in CI, and the new \"Wait for CoreDNS to be available\" check (correctly) timed out because no CoreDNS pod was running during the upgrade. This also lets CI exercise #323's CoreDNS patch.\n\nValidation: pre-commit (including ansible-lint) passes. The three-node `cluster-upgrade` Zuul job (1.33.4 \u2192 1.34.0) runs this path.\n\nStacked on #323: this PR targets that PR's branch, so the diff shows only this change. It moves to `main` once #323 merges.\n\n\n\n" change_url: https://github.com/vexxhost/ansible-collection-kubernetes/pull/324 commit_id: 41423f1a9031c1d89ff7894e0318dfbaded337e6 patchset: 41423f1a9031c1d89ff7894e0318dfbaded337e6 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/ansible-collection-kubernetes name: vexxhost/ansible-collection-kubernetes short_name: ansible-collection-kubernetes src_dir: src/github.com/vexxhost/ansible-collection-kubernetes src_dir: src/github.com/vexxhost/ansible-collection-kubernetes topic: null change: '324' change_message: "feat(kubernetes): upgrade control plane nodes one at a time with health checks\n\nBottom of the upgrade stack (#323 merged): this PR targets `main`, and the rest of the stack builds on it. Replaces #314, which GitHub closed as merged by mistake when the stack was reordered.\n\nDuring a control plane upgrade, two etcd members can be down at once. After `kubeadm upgrade apply`, the role runs `kubeadm upgrade node` on the other control plane nodes with `throttle: 1`, which only waits for the previous node's command to return. kubeadm returns once it has replaced the node's static pod manifests, while etcd, the API server, the controller manager and the scheduler are still restarting. The next node can then start restarting its own. On a three-node control plane that's two of three etcd members down together, which loses quorum.\n\nThis PR upgrades the control plane nodes in a loop (`control-plane-upgrade-node.yml`). After `kubeadm upgrade apply`, and after each node that needed an upgrade, it waits (`health-check.yml`) for:\n\n- the API's `/readyz` endpoint;\n- one `etcd`, `kube-apiserver`, `kube-controller-manager` and `kube-scheduler` pod per control plane node, all Ready;\n- the CoreDNS rollout.\n\nNew defaults: `kubernetes_health_check_components`, `kubernetes_health_check_retries` (30), `kubernetes_health_check_delay` (10) and `kubernetes_health_check_timeout` (`300s`).\n\n**CI change:** the molecule jobs now set `kubernetes_coredns_node_selector: {node-role.kubernetes.io/control-plane: \"\"}`. The default selector, `openstack-control-plane: enabled`, is an Atmosphere node label that CI nodes don't have. CoreDNS was never schedulable in CI, and the new \"Wait for CoreDNS to be available\" check (correctly) timed out because no CoreDNS pod was running during the upgrade. This also lets CI exercise #323's CoreDNS patch.\n\nValidation: pre-commit (including ansible-lint) passes. The three-node `cluster-upgrade` Zuul job (1.33.4 \u2192 1.34.0) runs this path.\n\nStacked on #323: this PR targets that PR's branch, so the diff shows only this change. It moves to `main` once #323 merges.\n\n\n\n" change_url: https://github.com/vexxhost/ansible-collection-kubernetes/pull/324 child_jobs: [] commit_id: 41423f1a9031c1d89ff7894e0318dfbaded337e6 event_id: be1527f0-d2e4-11f1-8a6b-5380fae1d774 executor: hostname: 2d72f0692154 inventory_file: /var/lib/zuul/builds/2e00dc2c478b41acb073b1087cb305dc/ansible/inventory.yaml log_root: /var/lib/zuul/builds/2e00dc2c478b41acb073b1087cb305dc/work/logs result_data_file: /var/lib/zuul/builds/2e00dc2c478b41acb073b1087cb305dc/work/results.json src_root: /var/lib/zuul/builds/2e00dc2c478b41acb073b1087cb305dc/work/src work_root: /var/lib/zuul/builds/2e00dc2c478b41acb073b1087cb305dc/work include_vars: [] items: - branch: main change: '324' change_message: "feat(kubernetes): upgrade control plane nodes one at a time with health checks\n\nBottom of the upgrade stack (#323 merged): this PR targets `main`, and the rest of the stack builds on it. Replaces #314, which GitHub closed as merged by mistake when the stack was reordered.\n\nDuring a control plane upgrade, two etcd members can be down at once. After `kubeadm upgrade apply`, the role runs `kubeadm upgrade node` on the other control plane nodes with `throttle: 1`, which only waits for the previous node's command to return. kubeadm returns once it has replaced the node's static pod manifests, while etcd, the API server, the controller manager and the scheduler are still restarting. The next node can then start restarting its own. On a three-node control plane that's two of three etcd members down together, which loses quorum.\n\nThis PR upgrades the control plane nodes in a loop (`control-plane-upgrade-node.yml`). After `kubeadm upgrade apply`, and after each node that needed an upgrade, it waits (`health-check.yml`) for:\n\n- the API's `/readyz` endpoint;\n- one `etcd`, `kube-apiserver`, `kube-controller-manager` and `kube-scheduler` pod per control plane node, all Ready;\n- the CoreDNS rollout.\n\nNew defaults: `kubernetes_health_check_components`, `kubernetes_health_check_retries` (30), `kubernetes_health_check_delay` (10) and `kubernetes_health_check_timeout` (`300s`).\n\n**CI change:** the molecule jobs now set `kubernetes_coredns_node_selector: {node-role.kubernetes.io/control-plane: \"\"}`. The default selector, `openstack-control-plane: enabled`, is an Atmosphere node label that CI nodes don't have. CoreDNS was never schedulable in CI, and the new \"Wait for CoreDNS to be available\" check (correctly) timed out because no CoreDNS pod was running during the upgrade. This also lets CI exercise #323's CoreDNS patch.\n\nValidation: pre-commit (including ansible-lint) passes. The three-node `cluster-upgrade` Zuul job (1.33.4 \u2192 1.34.0) runs this path.\n\nStacked on #323: this PR targets that PR's branch, so the diff shows only this change. It moves to `main` once #323 merges.\n\n\n\n" change_url: https://github.com/vexxhost/ansible-collection-kubernetes/pull/324 commit_id: 41423f1a9031c1d89ff7894e0318dfbaded337e6 patchset: 41423f1a9031c1d89ff7894e0318dfbaded337e6 project: canonical_hostname: github.com canonical_name: github.com/vexxhost/ansible-collection-kubernetes name: vexxhost/ansible-collection-kubernetes short_name: ansible-collection-kubernetes src_dir: src/github.com/vexxhost/ansible-collection-kubernetes topic: null job: pre-commit jobtags: [] max_attempts: 3 message: ZmVhdChrdWJlcm5ldGVzKTogdXBncmFkZSBjb250cm9sIHBsYW5lIG5vZGVzIG9uZSBhdCBhIHRpbWUgd2l0aCBoZWFsdGggY2hlY2tzCgpCb3R0b20gb2YgdGhlIHVwZ3JhZGUgc3RhY2sgKCMzMjMgbWVyZ2VkKTogdGhpcyBQUiB0YXJnZXRzIGBtYWluYCwgYW5kIHRoZSByZXN0IG9mIHRoZSBzdGFjayBidWlsZHMgb24gaXQuIFJlcGxhY2VzICMzMTQsIHdoaWNoIEdpdEh1YiBjbG9zZWQgYXMgbWVyZ2VkIGJ5IG1pc3Rha2Ugd2hlbiB0aGUgc3RhY2sgd2FzIHJlb3JkZXJlZC4KCkR1cmluZyBhIGNvbnRyb2wgcGxhbmUgdXBncmFkZSwgdHdvIGV0Y2QgbWVtYmVycyBjYW4gYmUgZG93biBhdCBvbmNlLiBBZnRlciBga3ViZWFkbSB1cGdyYWRlIGFwcGx5YCwgdGhlIHJvbGUgcnVucyBga3ViZWFkbSB1cGdyYWRlIG5vZGVgIG9uIHRoZSBvdGhlciBjb250cm9sIHBsYW5lIG5vZGVzIHdpdGggYHRocm90dGxlOiAxYCwgd2hpY2ggb25seSB3YWl0cyBmb3IgdGhlIHByZXZpb3VzIG5vZGUncyBjb21tYW5kIHRvIHJldHVybi4ga3ViZWFkbSByZXR1cm5zIG9uY2UgaXQgaGFzIHJlcGxhY2VkIHRoZSBub2RlJ3Mgc3RhdGljIHBvZCBtYW5pZmVzdHMsIHdoaWxlIGV0Y2QsIHRoZSBBUEkgc2VydmVyLCB0aGUgY29udHJvbGxlciBtYW5hZ2VyIGFuZCB0aGUgc2NoZWR1bGVyIGFyZSBzdGlsbCByZXN0YXJ0aW5nLiBUaGUgbmV4dCBub2RlIGNhbiB0aGVuIHN0YXJ0IHJlc3RhcnRpbmcgaXRzIG93bi4gT24gYSB0aHJlZS1ub2RlIGNvbnRyb2wgcGxhbmUgdGhhdCdzIHR3byBvZiB0aHJlZSBldGNkIG1lbWJlcnMgZG93biB0b2dldGhlciwgd2hpY2ggbG9zZXMgcXVvcnVtLgoKVGhpcyBQUiB1cGdyYWRlcyB0aGUgY29udHJvbCBwbGFuZSBub2RlcyBpbiBhIGxvb3AgKGBjb250cm9sLXBsYW5lLXVwZ3JhZGUtbm9kZS55bWxgKS4gQWZ0ZXIgYGt1YmVhZG0gdXBncmFkZSBhcHBseWAsIGFuZCBhZnRlciBlYWNoIG5vZGUgdGhhdCBuZWVkZWQgYW4gdXBncmFkZSwgaXQgd2FpdHMgKGBoZWFsdGgtY2hlY2sueW1sYCkgZm9yOgoKLSB0aGUgQVBJJ3MgYC9yZWFkeXpgIGVuZHBvaW50OwotIG9uZSBgZXRjZGAsIGBrdWJlLWFwaXNlcnZlcmAsIGBrdWJlLWNvbnRyb2xsZXItbWFuYWdlcmAgYW5kIGBrdWJlLXNjaGVkdWxlcmAgcG9kIHBlciBjb250cm9sIHBsYW5lIG5vZGUsIGFsbCBSZWFkeTsKLSB0aGUgQ29yZUROUyByb2xsb3V0LgoKTmV3IGRlZmF1bHRzOiBga3ViZXJuZXRlc19oZWFsdGhfY2hlY2tfY29tcG9uZW50c2AsIGBrdWJlcm5ldGVzX2hlYWx0aF9jaGVja19yZXRyaWVzYCAoMzApLCBga3ViZXJuZXRlc19oZWFsdGhfY2hlY2tfZGVsYXlgICgxMCkgYW5kIGBrdWJlcm5ldGVzX2hlYWx0aF9jaGVja190aW1lb3V0YCAoYDMwMHNgKS4KCioqQ0kgY2hhbmdlOioqIHRoZSBtb2xlY3VsZSBqb2JzIG5vdyBzZXQgYGt1YmVybmV0ZXNfY29yZWRuc19ub2RlX3NlbGVjdG9yOiB7bm9kZS1yb2xlLmt1YmVybmV0ZXMuaW8vY29udHJvbC1wbGFuZTogIiJ9YC4gVGhlIGRlZmF1bHQgc2VsZWN0b3IsIGBvcGVuc3RhY2stY29udHJvbC1wbGFuZTogZW5hYmxlZGAsIGlzIGFuIEF0bW9zcGhlcmUgbm9kZSBsYWJlbCB0aGF0IENJIG5vZGVzIGRvbid0IGhhdmUuIENvcmVETlMgd2FzIG5ldmVyIHNjaGVkdWxhYmxlIGluIENJLCBhbmQgdGhlIG5ldyAiV2FpdCBmb3IgQ29yZUROUyB0byBiZSBhdmFpbGFibGUiIGNoZWNrIChjb3JyZWN0bHkpIHRpbWVkIG91dCBiZWNhdXNlIG5vIENvcmVETlMgcG9kIHdhcyBydW5uaW5nIGR1cmluZyB0aGUgdXBncmFkZS4gVGhpcyBhbHNvIGxldHMgQ0kgZXhlcmNpc2UgIzMyMydzIENvcmVETlMgcGF0Y2guCgpWYWxpZGF0aW9uOiBwcmUtY29tbWl0IChpbmNsdWRpbmcgYW5zaWJsZS1saW50KSBwYXNzZXMuIFRoZSB0aHJlZS1ub2RlIGBjbHVzdGVyLXVwZ3JhZGVgIFp1dWwgam9iICgxLjMzLjQg4oaSIDEuMzQuMCkgcnVucyB0aGlzIHBhdGguCgpTdGFja2VkIG9uICMzMjM6IHRoaXMgUFIgdGFyZ2V0cyB0aGF0IFBSJ3MgYnJhbmNoLCBzbyB0aGUgZGlmZiBzaG93cyBvbmx5IHRoaXMgY2hhbmdlLiBJdCBtb3ZlcyB0byBgbWFpbmAgb25jZSAjMzIzIG1lcmdlcy4KCgoK patchset: 41423f1a9031c1d89ff7894e0318dfbaded337e6 pipeline: check playbook_context: playbook_projects: trusted/project_0/github.com/vexxhost/zuul-config: canonical_name: github.com/vexxhost/zuul-config checkout: main commit: 298983cd1253e6833abdb49d87d912527e0e6597 trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: be52461cd48abccbf95b13a58b2551ac673d8ad7 trusted/project_2/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: 4fd48825891421af0774bdd500fee36e111d80a6 untrusted/project_0/github.com/vexxhost/zuul-jobs: canonical_name: github.com/vexxhost/zuul-jobs checkout: main commit: 4fd48825891421af0774bdd500fee36e111d80a6 untrusted/project_1/github.com/vexxhost/zuul-config: canonical_name: github.com/vexxhost/zuul-config checkout: main commit: 298983cd1253e6833abdb49d87d912527e0e6597 untrusted/project_2/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: be52461cd48abccbf95b13a58b2551ac673d8ad7 playbooks: - path: untrusted/project_0/github.com/vexxhost/zuul-jobs/playbooks/pre-commit/run.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_1/zuul-jobs/roles - checkout: main checkout_description: playbook branch link_name: ansible/playbook_0/role_2/zuul-jobs link_target: untrusted/project_0/github.com/vexxhost/zuul-jobs role_path: ansible/playbook_0/role_2/zuul-jobs/roles post_playbooks: - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/post.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_0/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_0/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_0/role_2/zuul-jobs/roles - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/post-logs.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_1/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/post_playbook_1/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/post_playbook_1/role_2/zuul-jobs/roles pre_playbooks: - path: trusted/project_0/github.com/vexxhost/zuul-config/playbooks/base/pre.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_0/role_1/zuul-jobs/roles - checkout: main checkout_description: zuul branch link_name: ansible/pre_playbook_0/role_2/zuul-jobs link_target: trusted/project_2/github.com/vexxhost/zuul-jobs role_path: ansible/pre_playbook_0/role_2/zuul-jobs/roles - path: untrusted/project_0/github.com/vexxhost/zuul-jobs/playbooks/pre-commit/pre.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_1/role_1/zuul-jobs/roles - checkout: main checkout_description: playbook branch link_name: ansible/pre_playbook_1/role_2/zuul-jobs link_target: untrusted/project_0/github.com/vexxhost/zuul-jobs role_path: ansible/pre_playbook_1/role_2/zuul-jobs/roles post_review: false post_timeout: null pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/vexxhost/ansible-collection-kubernetes name: vexxhost/ansible-collection-kubernetes short_name: ansible-collection-kubernetes src_dir: src/github.com/vexxhost/ansible-collection-kubernetes projects: github.com/vexxhost/ansible-collection-kubernetes: canonical_hostname: github.com canonical_name: github.com/vexxhost/ansible-collection-kubernetes checkout: main checkout_description: zuul branch commit: dc162638c826c629e152151b237b492143e59dfa name: vexxhost/ansible-collection-kubernetes required: false short_name: ansible-collection-kubernetes src_dir: src/github.com/vexxhost/ansible-collection-kubernetes ref: refs/pull/324/head resources: {} tenant: oss timeout: 1800 topic: null voting: true