diff options
11 files changed, 199 insertions, 75 deletions
diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml new file mode 100644 index 000000000..6c12e8245 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml @@ -0,0 +1,105 @@ + +- name: Check for appropriate Docker versions for 1.9.x to 1.10.x upgrade + hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config + roles: + - openshift_facts + tasks: + - name: Determine available Docker version + script: ../../../../common/openshift-cluster/upgrades/files/rpm_versions.sh docker + register: g_docker_version_result + when: not openshift.common.is_atomic | bool + + - name: Check if Docker is installed + command: rpm -q docker + register: pkg_check + failed_when: pkg_check.rc > 1 + changed_when: no + when: not openshift.common.is_atomic | bool + + - set_fact: + g_docker_version: "{{ g_docker_version_result.stdout | from_yaml }}" + when: not openshift.common.is_atomic | bool + + - name: Set fact if docker requires an upgrade + set_fact: + docker_upgrade: true + when: not openshift.common.is_atomic | bool and pkg_check.rc == 0 and g_docker_version.curr_version | version_compare('1.10','<') + + - fail: + msg: This playbook requires access to Docker 1.10 or later + when: g_docker_version.avail_version | default(g_docker_version.curr_version, true) | version_compare('1.10','<') + +# If a node fails, halt everything, the admin will need to clean up and we +# don't want to carry on, potentially taking out every node. The playbook can safely be re-run +# and will not take any action on a node already running 1.10+. +- name: Evacuate and upgrade nodes + hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config + serial: 1 + any_errors_fatal: true + tasks: + - debug: var=docker_upgrade + + - name: Prepare for Node evacuation + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --schedulable=false + delegate_to: "{{ groups.oo_first_master.0 }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config + +# TODO: skip all node evac stuff for non-nodes (i.e. separate containerized etcd hosts) + - name: Evacuate Node for Kubelet upgrade + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --evacuate --force + delegate_to: "{{ groups.oo_first_master.0 }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config + + - name: Stop containerized services + service: name={{ item }} state=stopped + with_items: + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + - etcd + - openvswitch + failed_when: false + when: docker_upgrade is defined and docker_upgrade | bool and openshift.common.is_containerized | bool + + - name: Remove all containers and images + script: files/nuke_images.sh docker + register: nuke_images_result + when: docker_upgrade is defined and docker_upgrade | bool + + - name: Upgrade Docker + command: "{{ ansible_pkg_mgr}} update -y docker" + register: docker_upgrade_result + when: docker_upgrade is defined and docker_upgrade | bool + + - name: Restart containerized services + service: name={{ item }} state=started + with_items: + - etcd + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: docker_upgrade is defined and docker_upgrade | bool and openshift.common.is_containerized | bool + + - name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_masters_to_config + + - name: Set node schedulability + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --schedulable=true + delegate_to: "{{ groups.oo_first_master.0 }}" + when: openshift.node.schedulable | bool + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config and openshift.node.schedulable | bool + diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh b/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh new file mode 100644 index 000000000..9a5ee2276 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Stop any running containers +running_container_count=`docker ps -q | wc -l` +if test $running_container_count -gt 0 +then + docker stop $(docker ps -q) +fi + +# Delete all containers +container_count=`docker ps -a -q | wc -l` +if test $container_count -gt 0 +then + docker rm -f -v $(docker ps -a -q) +fi + +# Delete all images (forcefully) +image_count=`docker images -q | wc -l` +if test $image_count -gt 0 +then + # Taken from: https://gist.github.com/brianclements/f72b2de8e307c7b56689#gistcomment-1443144 + docker rmi $(docker images | grep "$2/\|/$2 \| $2 \|$2 \|$2-\|$2_" | awk '{print $1 ":" $2}') 2>/dev/null || echo "No images matching \"$2\" left to purge." +fi diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/roles b/playbooks/byo/openshift-cluster/upgrades/docker/roles new file mode 120000 index 000000000..6bc1a7aef --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/roles @@ -0,0 +1 @@ +../../../../../roles
\ No newline at end of file diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml new file mode 100644 index 000000000..0f86abd89 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml @@ -0,0 +1,29 @@ +# Playbook to upgrade Docker to the max allowable version for an OpenShift cluster. +# +# Currently only supports upgrading 1.9.x to >= 1.10.x. +- hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - include_vars: ../../cluster_hosts.yml + - add_host: + name: "{{ item }}" + groups: l_oo_all_hosts + with_items: g_all_hosts | default([]) + changed_when: false + +- hosts: l_oo_all_hosts + gather_facts: no + tasks: + - include_vars: ../../cluster_hosts.yml + +- include: ../../../../common/openshift-cluster/evaluate_groups.yml + vars: + # Do not allow adding hosts during upgrade. + g_new_master_hosts: [] + g_new_node_hosts: [] + openshift_cluster_id: "{{ cluster_id | default('default') }}" + openshift_deployment_type: "{{ deployment_type }}" + +- include: docker_upgrade.yml diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml index 24617620b..8c89e118e 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml @@ -4,10 +4,6 @@ become: no gather_facts: no tasks: - - name: Verify Ansible version is greater than or equal to 1.9.4 and less than 2.0 - fail: - msg: "Unsupported ansible version: {{ ansible_version }} found." - when: ansible_version.full | version_compare('1.9.4', 'lt') or ansible_version.full | version_compare('2.0', 'ge') - include_vars: ../../../../byo/openshift-cluster/cluster_hosts.yml - add_host: name: "{{ item }}" diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml deleted file mode 100644 index 9ade795f2..000000000 --- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml +++ /dev/null @@ -1,15 +0,0 @@ -- name: Check if Docker is installed - command: rpm -q docker - register: pkg_check - failed_when: pkg_check.rc > 1 - changed_when: no - -- name: Upgrade Docker - command: "{{ ansible_pkg_mgr}} update -y docker" - when: pkg_check.rc == 0 and g_docker_version.curr_version | version_compare('1.9','<') - register: docker_upgrade - -- name: Restart Docker - command: systemctl restart docker - when: docker_upgrade | changed - diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml index c93bf2a17..156e80c0f 100644 --- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml @@ -3,13 +3,13 @@ # The restart playbook should be run after this playbook completes. ############################################################################### -- name: Upgrade docker +- include: ../../../../byo/openshift-cluster/upgrades/docker/docker_upgrade.yml + +- name: Update Docker facts hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config roles: - openshift_facts tasks: - - include: docker_upgrade.yml - when: not openshift.common.is_atomic | bool - name: Set post docker install facts openshift_facts: role: "{{ item.role }}" diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index b9b2666fb..f002dbc01 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,41 +1,42 @@ --- -# tasks file for docker - -- name: Get current installed version if docker_version is specified +# Going forward we require Docker 1.10 or greater. If the user has a lesser version installed they must run a separate upgrade process. +- name: Get current installed Docker version command: "{{ repoquery_cmd }} --installed --qf '%{version}' docker" - when: not openshift.common.is_atomic | bool and docker_version != '' - register: docker_version_result + when: not openshift.common.is_atomic | bool + register: curr_docker_version changed_when: false -- stat: path=/etc/sysconfig/docker-storage - register: docker_storage_check +# TODO: The use of upgrading var will be removed in the coming upgrade refactor. This is a temporary +# fix to wory around the fact that right now, this role is called during upgrade, before we're +# ready to upgrade Docker. +- name: Fail if Docker upgrade is required + fail: + msg: "Docker {{ curr_docker_version.stdout }} must be upgraded to Docker 1.10 or greater" + when: not upgrading | bool and not curr_docker_version | skipped and curr_docker_version.stdout | default('0.0', True) | version_compare('1.10', '<') -- name: Remove deferred deletion for downgrades from 1.9 +- name: Get latest available version of Docker command: > - sed -i 's/--storage-opt dm.use_deferred_deletion=true//' /etc/sysconfig/docker-storage - when: docker_storage_check.stat.exists | bool and not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare('1.9', '>=') and docker_version | version_compare('1.9', '<') - -- name: Downgrade docker if necessary - command: "{{ ansible_pkg_mgr }} swap -y docker-* docker-*{{ docker_version }}" - register: docker_downgrade_result - when: not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare(docker_version, 'gt') + {{ repoquery_cmd }} --qf '%{version}' "docker" + register: avail_docker_version + failed_when: false + changed_when: false + when: not openshift.common.is_atomic | bool -- name: Install docker - action: "{{ ansible_pkg_mgr }} name=docker{{ '-' + docker_version if docker_version is defined and docker_version != '' else '' }} state=present" - when: not openshift.common.is_atomic | bool and docker_downgrade_result | skipped +- name: Verify Docker >= 1.10 is available + fail: + msg: "Docker {{ avail_docker_version.stdout }} is available, but 1.10 or greater is required" + when: not avail_docker_version | skipped and avail_docker_version.stdout | default('0.0', True) | version_compare('1.10', '<') -# If docker were enabled and started before we downgraded it may have entered a -# failed state. Check for that and clear it if necessary. -- name: Check that docker hasn't entered failed state - command: systemctl show docker - register: docker_state - changed_when: False +- stat: path=/etc/sysconfig/docker-storage + register: docker_storage_check -- name: Reset docker service state - command: systemctl reset-failed docker.service - when: " 'ActiveState=failed' in docker_state.stdout " +# Make sure Docker is installed, but does not update a running version. +# Docker upgrades are handled by a separate playbook. +- name: Install Docker + action: "{{ ansible_pkg_mgr }} name=docker state=present" + when: not openshift.common.is_atomic | bool -- name: Start the docker service +- name: Start the Docker service service: name: docker enabled: yes @@ -86,7 +87,7 @@ notify: - restart docker -- name: Set various docker options +- name: Set various Docker options lineinfile: dest: /etc/sysconfig/docker regexp: '^OPTIONS=.*$' diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 71735dc25..a798dc973 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -28,18 +28,18 @@ state: directory mode: 0700 -- name: Check for etcd service presence - command: systemctl show etcd.service - register: etcd_show - changed_when: false - - name: Disable system etcd when containerized - when: etcd_is_containerized | bool and 'LoadState=not-found' not in etcd_show.stdout + when: etcd_is_containerized | bool service: name: etcd state: stopped enabled: no +- name: Check for etcd service presence + command: systemctl show etcd.service + register: etcd_show + changed_when: false + - name: Mask system etcd when containerized when: etcd_is_containerized | bool and 'LoadState=not-found' not in etcd_show.stdout command: systemctl mask etcd diff --git a/roles/openshift_docker_facts/tasks/main.yml b/roles/openshift_docker_facts/tasks/main.yml index 43359dcb5..cdea90413 100644 --- a/roles/openshift_docker_facts/tasks/main.yml +++ b/roles/openshift_docker_facts/tasks/main.yml @@ -57,14 +57,3 @@ l_common_version: "{{ common_version.stdout | default('0.0', True) }}" when: not openshift.common.is_containerized | bool -- name: Set docker version to be installed - set_fact: - docker_version: "{{ '1.8.2' }}" - when: " ( l_common_version | version_compare('3.2','<') and openshift.common.service_type in ['openshift', 'atomic-openshift'] ) or - ( l_common_version | version_compare('1.1.4','<') and openshift.common.service_type == 'origin' )" - -- name: Set docker version to be installed - set_fact: - docker_version: "{{ '1.9.1' }}" - when: " ( l_common_version | version_compare('3.2','>') and openshift.common.service_type == 'atomic-openshift' ) or - ( l_common_version | version_compare('1.2','>') and openshift.common.service_type == 'origin' )" diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index f70eaf144..63a54a0d9 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -164,14 +164,9 @@ register: start_result notify: Verify API Server -- name: Check for non-HA master service presence - command: systemctl show {{ openshift.common.service_type }}-master.service - register: master_svc_show - changed_when: false - -- name: Stop and disable non-HA master when running HA +- name: Stop and disable non HA master when running HA service: name={{ openshift.common.service_type }}-master enabled=no state=stopped - when: openshift_master_ha | bool and 'LoadState=not-found' not in master_svc_show.stdout + when: openshift_master_ha | bool - set_fact: master_service_status_changed: "{{ start_result | changed }}" |