1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
|
---
###############################################################################
# Evaluate host groups and gather facts
###############################################################################
- name: Load openshift_facts and update repos
hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config:oo_lb_to_config
roles:
- openshift_facts
- openshift_repos
- name: Evaluate additional groups for upgrade
hosts: localhost
connection: local
become: no
tasks:
- name: Evaluate etcd_hosts_to_backup
add_host:
name: "{{ item }}"
groups: etcd_hosts_to_backup
with_items: groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master
###############################################################################
# Pre-upgrade checks
###############################################################################
- name: Verify upgrade can proceed on first master
hosts: oo_first_master
vars:
target_version: "{{ '1.2' if deployment_type == 'origin' else '3.2' }}"
g_pacemaker_upgrade_url_segment: "{{ 'org/latest' if deployment_type =='origin' else '.com/enterprise/3.1' }}"
gather_facts: no
tasks:
- fail:
msg: >
This upgrade is only supported for atomic-enterprise, origin, openshift-enterprise, and online
deployment types
when: deployment_type not in ['atomic-enterprise', 'origin','openshift-enterprise', 'online']
- fail:
msg: >
This upgrade does not support Pacemaker:
https://docs.openshift.{{ g_pacemaker_upgrade_url_segment }}/install_config/upgrading/pacemaker_to_native_ha.html
when: openshift.master.cluster_method is defined and openshift.master.cluster_method == 'pacemaker'
# Error out in situations where the user has older versions specified in their
# inventory in any of the openshift_release, openshift_image_tag, and
# openshift_pkg_version variables. These must be removed or updated to proceed
# with upgrade.
# TODO: Should we block if you're *over* the next major release version as well?
- fail:
msg: >
openshift_pkg_version is {{ openshift_pkg_version }} which is not a
valid version for a {{ target_version }} upgrade
when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(target_version ,'<')
- fail:
msg: >
openshift_image_tag is {{ openshift_image_tag }} which is not a
valid version for a {{ target_version }} upgrade
when: openshift_image_tag is defined and openshift_image_tag.split('v',1).1 | version_compare(target_version ,'<')
- fail:
msg: >
openshift_release is {{ openshift_release }} which is not a
valid release for a {{ target_version }} upgrade
when: openshift_release is defined and not openshift_release | version_compare(target_version ,'=')
- include: ../../../../common/openshift-cluster/initialize_openshift_version.yml
vars:
# Request openshift_release 3.2 and let the openshift_version role handle converting this
# to a more specific version, respecting openshift_image_tag and openshift_pkg_version if
# defined, and overriding the normal behavior of protecting the installed version
openshift_release: "3.2"
openshift_protect_installed_version: False
- name: Verify master processes
hosts: oo_masters_to_config
roles:
- openshift_facts
tasks:
- openshift_facts:
role: master
local_facts:
ha: "{{ groups.oo_masters_to_config | length > 1 }}"
- name: Ensure Master is running
service:
name: "{{ openshift.common.service_type }}-master"
state: started
enabled: yes
when: openshift.master.ha is defined and not openshift.master.ha | bool and openshift.common.is_containerized | bool
- name: Ensure HA Master is running
service:
name: "{{ openshift.common.service_type }}-master-api"
state: started
enabled: yes
when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool
- name: Ensure HA Master is running
service:
name: "{{ openshift.common.service_type }}-master-controllers"
state: started
enabled: yes
when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool
- name: Verify node processes
hosts: oo_nodes_to_config
roles:
- openshift_facts
- openshift_docker_facts
tasks:
- name: Ensure Node is running
service:
name: "{{ openshift.common.service_type }}-node"
state: started
enabled: yes
when: openshift.common.is_containerized | bool
- name: Verify upgrade targets
hosts: oo_masters_to_config:oo_nodes_to_config
vars:
target_version: "{{ '1.2' if deployment_type == 'origin' else '3.2' }}"
openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}"
pre_tasks:
- fail:
msg: Verify OpenShift is already installed
when: openshift.common.version is not defined
- fail:
msg: Verify the correct version was found
when: verify_upgrade_version is defined and openshift_version != verify_upgrade_version
- name: Clean package cache
command: "{{ ansible_pkg_mgr }} clean all"
when: not openshift.common.is_atomic | bool
- set_fact:
g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}"
when: not openshift.common.is_containerized | bool
- name: Verify containers are available for upgrade
command: >
docker pull {{ openshift.common.cli_image }}:{{ openshift_image_tag }}
when: openshift.common.is_containerized | bool
- set_fact:
repoquery_cmd: "{{ 'dnf repoquery --latest-limit 1 -d 0' if ansible_pkg_mgr == 'dnf' else 'repoquery' }}"
when: not openshift.common.is_containerized | bool
- name: Check latest available OpenShift RPM version
command: >
{{ repoquery_cmd }} --qf '%{version}' "{{ openshift.common.service_type }}"
failed_when: false
changed_when: false
register: avail_openshift_version
when: not openshift.common.is_containerized | bool
- debug: var=avail_openshift_version
- name: Verify OpenShift 3.2 RPMs are available for upgrade
fail:
msg: "OpenShift {{ avail_openshift_version.stdout }} is available, but 3.2 or greater is required"
when: not openshift.common.is_containerized | bool and not avail_openshift_version | skipped and avail_openshift_version.stdout | default('0.0', True) | version_compare('3.2', '<')
# TODO: Are these two grep checks necessary anymore?
# Note: the version number is hardcoded here in hopes of catching potential
# bugs in how g_aos_versions.curr_version is set
- name: Verifying the correct version is installed for upgrade
shell: grep 3.1.1.6 {{ item }}
with_items:
- /etc/sysconfig/openvswitch
- /etc/sysconfig/{{ openshift.common.service_type }}*
when: verify_upgrade_version is defined
- name: Verifying the image version is used in the systemd unit
shell: grep IMAGE_VERSION {{ item }}
with_items:
- /etc/systemd/system/openvswitch.service
- /etc/systemd/system/{{ openshift.common.service_type }}*.service
when: openshift.common.is_containerized | bool and verify_upgrade_version is defined
- fail:
msg: This upgrade playbook must be run on Origin 1.1 or later
when: deployment_type == 'origin' and openshift.common.version | version_compare('1.1','<')
- fail:
msg: This upgrade playbook must be run on OpenShift Enterprise 3.1 or later
when: deployment_type == 'atomic-openshift' and openshift.common.version | version_compare('3.1','<')
- name: Determine available Docker
script: ../files/rpm_versions.sh docker
register: g_docker_version_result
when: not openshift.common.is_atomic | bool
- name: Determine available Docker
shell: "rpm -q --queryformat '---\ncurr_version: %{VERSION}\navail_version: \n' docker"
register: g_atomic_docker_version_result
when: openshift.common.is_atomic | bool
- set_fact:
g_docker_version: "{{ g_docker_version_result.stdout | from_yaml }}"
when: not openshift.common.is_atomic | bool
- set_fact:
g_docker_version: "{{ g_atomic_docker_version_result.stdout | from_yaml }}"
when: openshift.common.is_atomic | bool
# TODO: Update to 1.10 once branch merges
- fail:
msg: This playbook requires access to Docker 1.9 or later
when: g_docker_version.avail_version | default(g_docker_version.curr_version, true) | version_compare('1.9','<')
# TODO: add check to upgrade ostree to get latest Docker
- set_fact:
pre_upgrade_complete: True
##############################################################################
# Gate on pre-upgrade checks
##############################################################################
- name: Gate on pre-upgrade checks
hosts: localhost
connection: local
become: no
vars:
pre_upgrade_hosts: "{{ groups.oo_masters_to_config | union(groups.oo_nodes_to_config) }}"
tasks:
- set_fact:
pre_upgrade_completed: "{{ hostvars
| oo_select_keys(pre_upgrade_hosts)
| oo_collect('inventory_hostname', {'pre_upgrade_complete': true}) }}"
- set_fact:
pre_upgrade_failed: "{{ pre_upgrade_hosts | difference(pre_upgrade_completed) }}"
- fail:
msg: "Upgrade cannot continue. The following hosts did not complete pre-upgrade checks: {{ pre_upgrade_failed | join(',') }}"
when: pre_upgrade_failed | length > 0
###############################################################################
# Backup etcd
###############################################################################
- name: Backup etcd
hosts: etcd_hosts_to_backup
vars:
embedded_etcd: "{{ openshift.master.embedded_etcd }}"
timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
roles:
- openshift_facts
tasks:
# Ensure we persist the etcd role for this host in openshift_facts
- openshift_facts:
role: etcd
local_facts: {}
when: "'etcd' not in openshift"
- stat: path=/var/lib/openshift
register: var_lib_openshift
- stat: path=/var/lib/origin
register: var_lib_origin
- name: Create origin symlink if necessary
file: src=/var/lib/openshift/ dest=/var/lib/origin state=link
when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False
# TODO: replace shell module with command and update later checks
# We assume to be using the data dir for all backups.
- name: Check available disk space for etcd backup
shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1
register: avail_disk
# TODO: replace shell module with command and update later checks
- name: Check current embedded etcd disk usage
shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1
register: etcd_disk_usage
when: embedded_etcd | bool
- name: Abort if insufficient disk space for etcd backup
fail:
msg: >
{{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup,
{{ avail_disk.stdout }} Kb available.
when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
- name: Install etcd (for etcdctl)
action: "{{ ansible_pkg_mgr }} name=etcd state=latest"
when: not openshift.common.is_atomic | bool
- name: Generate etcd backup
command: >
etcdctl backup --data-dir={{ openshift.etcd.etcd_data_dir }}
--backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
- set_fact:
etcd_backup_complete: True
- name: Display location of etcd backup
debug:
msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
##############################################################################
# Gate on etcd backup
##############################################################################
- name: Gate on etcd backup
hosts: localhost
connection: local
become: no
tasks:
- set_fact:
etcd_backup_completed: "{{ hostvars
| oo_select_keys(groups.etcd_hosts_to_backup)
| oo_collect('inventory_hostname', {'etcd_backup_complete': true}) }}"
- set_fact:
etcd_backup_failed: "{{ groups.etcd_hosts_to_backup | difference(etcd_backup_completed) }}"
- fail:
msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}"
when: etcd_backup_failed | length > 0
|
