summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDevan Goodwin <dgoodwin@redhat.com>2015-12-14 15:29:24 -0400
committerAndrew Butcher <abutcher@redhat.com>2016-01-11 16:04:36 -0500
commitdea9abfe22864cf10d85d85370b1633ca18060b6 (patch)
tree88f264d80f0c20d2f171724a5fd63508794d021b
parent52a4f52d023c1fd670b761974e7a5f0ef35a0eaf (diff)
downloadopenshift-dea9abfe22864cf10d85d85370b1633ca18060b6.tar.gz
openshift-dea9abfe22864cf10d85d85370b1633ca18060b6.tar.bz2
openshift-dea9abfe22864cf10d85d85370b1633ca18060b6.tar.xz
openshift-dea9abfe22864cf10d85d85370b1633ca18060b6.zip
Implement simple master rolling restarts.
Blocks running ansible on a host that will be restarted. Can restart just services, or optionally the full system.
-rw-r--r--playbooks/byo/openshift-cluster/restart.yml7
-rw-r--r--playbooks/common/openshift-cluster/restart.yml78
2 files changed, 85 insertions, 0 deletions
diff --git a/playbooks/byo/openshift-cluster/restart.yml b/playbooks/byo/openshift-cluster/restart.yml
new file mode 100644
index 000000000..da0da69a6
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/restart.yml
@@ -0,0 +1,7 @@
+---
+- include: ../../common/openshift-cluster/restart.yml
+ vars:
+ g_etcd_hosts: "{{ groups.etcd | default([]) }}"
+ g_master_hosts: "{{ groups.masters | default([]) }}"
+ g_node_hosts: "{{ groups.nodes | default([]) }}"
+ g_lb_hosts: "{{ groups.lb | default([]) }}"
diff --git a/playbooks/common/openshift-cluster/restart.yml b/playbooks/common/openshift-cluster/restart.yml
new file mode 100644
index 000000000..4117f7297
--- /dev/null
+++ b/playbooks/common/openshift-cluster/restart.yml
@@ -0,0 +1,78 @@
+---
+- include: evaluate_groups.yml
+# TODO: verify this is an HA environment
+# TODO: fork for pacemaker vs haproxy (based on?)
+
+- name: Validate configuration for rolling restart
+ hosts: oo_masters_to_config
+ tasks:
+ - set_fact:
+ openshift_rolling_restart_mode: "{{ openshift_rolling_restart_mode | default('services') }}"
+ - fail:
+ msg: "openshift_rolling_restart_mode must be set to either 'services' or 'system'"
+ when: openshift_rolling_restart_mode is defined and openshift_rolling_restart_mode not in ["services", "system"]
+
+# Creating a temp file on localhost, we then check each system that will
+# be rebooted to see if that file exists, if so we know we're running
+# ansible on a machine that needs a reboot, and we need to error out.
+- name: Create temp file on localhost
+ hosts: localhost
+ connection: local
+ become: no
+ gather_facts: no
+ tasks:
+ - local_action: command mktemp
+ register: mktemp
+ changed_when: False
+
+- name: Check if temp file exists on any masters
+ hosts: oo_masters_to_config
+ tasks:
+ - stat: path="{{ hostvars.localhost.mktemp.stdout }}"
+ register: exists
+
+- name: Cleanup temp file on localhost
+ hosts: localhost
+ connection: local
+ become: no
+ gather_facts: no
+ tasks:
+ - file: path="{{ hostvars.localhost.mktemp.stdout }}" state=absent
+
+- name: Fail if restarting the system where ansible is running
+ hosts: oo_masters_to_config
+ any_errors_fatal: true
+ tasks:
+ - fail: msg="Cannot run playbook on a host that will be restarted."
+ when: exists.stat.exists
+
+- name: Restart Masters
+ hosts: oo_masters_to_config
+ serial: 1
+ roles:
+ - openshift_facts
+ tasks:
+ - name: Restart master system
+ # https://github.com/ansible/ansible/issues/10616
+ shell: sleep 2 && shutdown -r now "OpenShift Ansible master rolling restart"
+ async: 1
+ poll: 0
+ ignore_errors: true
+ become: yes
+ when: openshift_rolling_restart_mode == 'system'
+ - name: Restart master services
+ service:
+ name: "{{ openshift.common.service_type }}-master-api"
+ state: restarted
+ # NOTE: no need to check openshift_master_ha here, we know it must be,
+ # thus the api service is the one we restart.
+ when: openshift_rolling_restart_mode == 'services'
+
+ - name: Wait for master API to come back online
+ become: no
+ local_action:
+ module: wait_for
+ host="{{ inventory_hostname }}"
+ state=started
+ delay=10
+ port=8443 # TODO: should this be made a master host variable?