1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
|
---
###############################################################################
# Evaluate host groups and gather facts
###############################################################################
- name: Load openshift_facts
hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config:oo_lb_to_config
roles:
- openshift_facts
- name: Evaluate additional groups for upgrade
hosts: localhost
connection: local
become: no
tasks:
- name: Evaluate etcd_hosts_to_backup
add_host:
name: "{{ item }}"
groups: etcd_hosts_to_backup
with_items: groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master
###############################################################################
# Pre-upgrade checks
###############################################################################
- name: Verify upgrade can proceed
hosts: oo_first_master
vars:
target_version: "{{ '1.2' if deployment_type == 'origin' else '3.1.1.900' }}"
gather_facts: no
tasks:
- fail:
msg: >
This upgrade is only supported for atomic-enterprise, origin, openshift-enterprise, and online
deployment types
when: deployment_type not in ['atomic-enterprise', 'origin','openshift-enterprise', 'online']
- fail:
msg: >
openshift_pkg_version is {{ openshift_pkg_version }} which is not a
valid version for a {{ target_version }} upgrade
when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(target_version ,'<')
- fail:
msg: >
openshift_image_tag is {{ openshift_image_tag }} which is not a
valid version for a {{ target_version }} upgrade
when: openshift_image_tag is defined and openshift_image_tag.split('v',1).1 | version_compare(target_version ,'<')
- name: Verify upgrade can proceed
hosts: oo_masters_to_config
roles:
- openshift_facts
tasks:
- name: Ensure Master is running
service:
name: "{{ openshift.common.service_type }}-master"
state: started
enabled: yes
when: openshift.master.ha is defined and not openshift.master.ha | bool and openshift.common.is_containerized | bool
- name: Ensure HA Master is running
service:
name: "{{ openshift.common.service_type }}-master-api"
state: started
enabled: yes
when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool
- name: Ensure HA Master is running
service:
name: "{{ openshift.common.service_type }}-master-controllers"
state: started
enabled: yes
when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool
post_tasks:
- openshift_facts:
role: master
local_facts:
ha: "{{ groups.oo_masters_to_config | length > 1 }}"
- name: Verify upgrade can proceed
hosts: oo_nodes_to_config
roles:
- openshift_facts
tasks:
- name: Ensure Node is running
service:
name: "{{ openshift.common.service_type }}-node"
state: started
enabled: yes
when: openshift.common.is_containerized | bool
- name: Verify upgrade can proceed
hosts: oo_masters_to_config:oo_nodes_to_config
vars:
target_version: "{{ '1.2' if deployment_type == 'origin' else '3.1.1.900' }}"
openshift_docker_hosted_registry_insecure: True
openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.master.portal_net }}"
handlers:
- include: ../../../../../roles/openshift_master/handlers/main.yml
- include: ../../../../../roles/openshift_node/handlers/main.yml
roles:
- openshift_cli
tasks:
- name: Clean package cache
command: "{{ ansible_pkg_mgr }} clean all"
when: not openshift.common.is_atomic | bool
- set_fact:
g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}"
when: not openshift.common.is_containerized | bool
- name: Determine available versions
script: ../files/rpm_versions.sh {{ g_new_service_name }}
register: g_rpm_versions_result
when: not openshift.common.is_containerized | bool
- set_fact:
g_aos_versions: "{{ g_rpm_versions_result.stdout | from_yaml }}"
when: not openshift.common.is_containerized | bool
- name: Determine available versions
script: ../files/openshift_container_versions.sh {{ openshift.common.service_type }}
register: g_containerized_versions_result
when: openshift.common.is_containerized | bool
- set_fact:
g_aos_versions: "{{ g_containerized_versions_result.stdout | from_yaml }}"
when: openshift.common.is_containerized | bool
- set_fact:
g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}"
when: openshift_pkg_version is not defined
- set_fact:
g_new_version: "{{ openshift_pkg_version | replace('-','') }}"
when: openshift_pkg_version is defined
- set_fact:
g_new_version: "{{ openshift_image_tag | replace('v','') }}"
when: openshift_image_tag is defined
- fail:
msg: Verifying the correct version was found
when: verify_upgrade_version is defined and g_new_version != verify_upgrade_version
- include_vars: ../../../../../roles/openshift_master/vars/main.yml
when: inventory_hostname in groups.oo_masters_to_config
- name: Update systemd units
include: ../../../../../roles/openshift_master/tasks/systemd_units.yml openshift_version=g_aos_versions.curr_version
when: inventory_hostname in groups.oo_masters_to_config
- include_vars: ../../../../../roles/openshift_node/vars/main.yml
when: inventory_hostname in groups.oo_nodes_to_config
- name: Update systemd units
include: ../../../../../roles/openshift_node/tasks/systemd_units.yml openshift_version=g_aos_versions.curr_version
when: inventory_hostname in groups.oo_nodes_to_config
# Note: the version number is hardcoded here in hopes of catching potential
# bugs in how g_aos_versions.curr_version is set
- name: Verifying the correct version is installed for upgrade
shell: grep 3.1.1.6 {{ item }}
with_items:
- /etc/sysconfig/openvswitch
- /etc/sysconfig/{{ openshift.common.service_type }}*
when: verify_upgrade_version is defined
- name: Verifying the image version is used in the systemd unit
shell: grep IMAGE_VERSION {{ item }}
with_items:
- /etc/systemd/system/openvswitch.service
- /etc/systemd/system/{{ openshift.common.service_type }}*.service
when: openshift.common.is_containerized | bool
- fail:
msg: This playbook requires Origin 1.1 or later
when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.1','<')
- fail:
msg: This playbook requires Atomic Enterprise Platform/OpenShift Enterprise 3.1 or later
when: deployment_type == 'atomic-openshift' and g_aos_versions.curr_version | version_compare('3.1','<')
- fail:
msg: Upgrade packages not found
when: openshift_image_tag is not defined and (g_aos_versions.avail_version | default(g_aos_versions.curr_version, true) | version_compare(target_version, '<'))
- name: Determine available Docker
script: ../files/rpm_versions.sh docker
register: g_docker_version_result
when: not openshift.common.is_atomic | bool
- name: Determine available Docker
shell: "rpm -q --queryformat '---\ncurr_version: %{VERSION}\navail_version: \n' docker"
register: g_atomic_docker_version_result
when: openshift.common.is_atomic | bool
- set_fact:
g_docker_version: "{{ g_docker_version_result.stdout | from_yaml }}"
when: not openshift.common.is_atomic | bool
- set_fact:
g_docker_version: "{{ g_atomic_docker_version_result.stdout | from_yaml }}"
when: openshift.common.is_atomic | bool
- fail:
msg: This playbook requires access to Docker 1.9 or later
when: not openshift.common.is_atomic | bool
and (g_docker_version.avail_version | default(g_docker_version.curr_version, true) | version_compare('1.9','<'))
# TODO: add check to upgrade ostree to get latest Docker
- set_fact:
pre_upgrade_complete: True
##############################################################################
# Gate on pre-upgrade checks
##############################################################################
- name: Gate on pre-upgrade checks
hosts: localhost
connection: local
become: no
vars:
pre_upgrade_hosts: "{{ groups.oo_masters_to_config | union(groups.oo_nodes_to_config) }}"
tasks:
- set_fact:
pre_upgrade_completed: "{{ hostvars
| oo_select_keys(pre_upgrade_hosts)
| oo_collect('inventory_hostname', {'pre_upgrade_complete': true}) }}"
- set_fact:
pre_upgrade_failed: "{{ pre_upgrade_hosts | difference(pre_upgrade_completed) }}"
- fail:
msg: "Upgrade cannot continue. The following hosts did not complete pre-upgrade checks: {{ pre_upgrade_failed | join(',') }}"
when: pre_upgrade_failed | length > 0
###############################################################################
# Backup etcd
###############################################################################
- name: Backup etcd
hosts: etcd_hosts_to_backup
vars:
embedded_etcd: "{{ openshift.master.embedded_etcd }}"
timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
roles:
- openshift_facts
tasks:
# Ensure we persist the etcd role for this host in openshift_facts
- openshift_facts:
role: etcd
local_facts: {}
when: "'etcd' not in openshift"
- stat: path=/var/lib/openshift
register: var_lib_openshift
- stat: path=/var/lib/origin
register: var_lib_origin
- name: Create origin symlink if necessary
file: src=/var/lib/openshift/ dest=/var/lib/origin state=link
when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False
# TODO: replace shell module with command and update later checks
# We assume to be using the data dir for all backups.
- name: Check available disk space for etcd backup
shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1
register: avail_disk
# TODO: replace shell module with command and update later checks
- name: Check current embedded etcd disk usage
shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1
register: etcd_disk_usage
when: embedded_etcd | bool
- name: Abort if insufficient disk space for etcd backup
fail:
msg: >
{{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup,
{{ avail_disk.stdout }} Kb available.
when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
- name: Install etcd (for etcdctl)
action: "{{ ansible_pkg_mgr }} name=etcd state=latest"
when: not openshift.common.is_atomic | bool
- name: Generate etcd backup
command: >
etcdctl backup --data-dir={{ openshift.etcd.etcd_data_dir }}
--backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
- set_fact:
etcd_backup_complete: True
- name: Display location of etcd backup
debug:
msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
##############################################################################
# Gate on etcd backup
##############################################################################
- name: Gate on etcd backup
hosts: localhost
connection: local
become: no
tasks:
- set_fact:
etcd_backup_completed: "{{ hostvars
| oo_select_keys(groups.etcd_hosts_to_backup)
| oo_collect('inventory_hostname', {'etcd_backup_complete': true}) }}"
- set_fact:
etcd_backup_failed: "{{ groups.etcd_hosts_to_backup | difference(etcd_backup_completed) }}"
- fail:
msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}"
when: etcd_backup_failed | length > 0
|
