From 006f333828db373435daa15483d2ab753048f62a Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Tue, 13 Mar 2018 05:01:30 +0100 Subject: Production --- roles/ands_facts/defaults/main.yml | 6 ++- roles/ands_facts/tasks/find_interface_by_net.yml | 17 ++++++ roles/ands_facts/tasks/network.yml | 33 ++++++++---- roles/ands_network/defaults/main.yml | 1 + roles/ands_network/tasks/nm_configure.yml | 39 ++++++++++++++ .../ands_network/tasks/nm_configure_connection.yml | 51 ++++++++++++++++++ roles/docker/defaults/main.yml | 2 + roles/docker/tasks/configure.yml | 63 ++++++++++++++++++++++ roles/docker/tasks/install.yml | 13 +++++ roles/docker/tasks/main.yml | 58 +------------------- roles/docker/tasks/storage.yml | 34 ++++++++++-- roles/ofed/tasks/main.yml | 6 +++ 12 files changed, 251 insertions(+), 72 deletions(-) create mode 100644 roles/ands_facts/tasks/find_interface_by_net.yml create mode 100644 roles/ands_network/defaults/main.yml create mode 100644 roles/ands_network/tasks/nm_configure.yml create mode 100644 roles/ands_network/tasks/nm_configure_connection.yml create mode 100644 roles/docker/tasks/configure.yml create mode 100644 roles/docker/tasks/install.yml create mode 100644 roles/ofed/tasks/main.yml (limited to 'roles') diff --git a/roles/ands_facts/defaults/main.yml b/roles/ands_facts/defaults/main.yml index c74984e..5feb354 100644 --- a/roles/ands_facts/defaults/main.yml +++ b/roles/ands_facts/defaults/main.yml @@ -25,7 +25,9 @@ ands_cluster_domain: "{{ ansible_domain }}" ands_inner_domain: "{{ ands_cluster_domain }}" ands_default_ip: "{{ ansible_default_ipv4.address }}" +ands_default_cidr: "{{ ansible_default_ipv4.address ~ '/' ~ ( ansible_default_ipv4.netmask | ipaddr('prefix') ) }}" ands_openshift_default_ip: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_ip, ands_none) }}" +ands_openshift_default_cidr: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_cidr, ands_none) }}" ands_openshift_default_hostname: "{{ (ands_hostname_template is defined) | ternary(ands_hostname_template ~ ands_host_id, ansible_hostname) }}" ands_inner_lb: true @@ -33,5 +35,5 @@ ands_use_inner_lb: false ands_inner_lb_id: 254 ands_inner_lb_hostname: 'ands-lb' -#ands_openshift_inner_interface: -#ands_openshift_public_interface: +#ands_inner_interface: +#ands_public_interface: diff --git a/roles/ands_facts/tasks/find_interface_by_net.yml b/roles/ands_facts/tasks/find_interface_by_net.yml new file mode 100644 index 0000000..ad44578 --- /dev/null +++ b/roles/ands_facts/tasks/find_interface_by_net.yml @@ -0,0 +1,17 @@ +- name: "Looking for interface holding {{ net }}" + set_fact: + "{{ var }}": "{{ eth['device'] }}" + vars: + eth: "{{ hostvars[inventory_hostname]['ansible_' + item] | default({}) }}" + ipv4: "{{ eth['ipv4'] | default({}) }}" + q: "{{ eth | json_query('ipv4_secondaries[*].network') }}" + sec: "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}" + nets: "{{ sec | union([ipv4.network]) }}" + when: + - eth['type'] is defined + - eth['ipv4'] is defined + - eth['device'] is defined + - eth['type'] == 'ether' + - net | ipaddr('network') in nets + with_items: + - "{{ hostvars[inventory_hostname]['ansible_interfaces'] }}" diff --git a/roles/ands_facts/tasks/network.yml b/roles/ands_facts/tasks/network.yml index 808d7b6..1acafc1 100644 --- a/roles/ands_facts/tasks/network.yml +++ b/roles/ands_facts/tasks/network.yml @@ -12,10 +12,13 @@ ands_use_inner_lb: "{{ ands_use_inner_lb }}" ands_inner_lb_ip: "{{ ands_openshift_network | ipaddr(ands_inner_lb_id) | ipaddr('address') }}" ands_inner_lb_hostname: "{{ ands_inner_lb_hostname }}" + ands_openshift_cidr: "{{ ands_openshift_network | ipaddr(ands_host_id) }}" ands_openshift_ip: "{{ ands_openshift_network | ipaddr(ands_host_id) | ipaddr('address') }}" ands_openshift_hostname: "{{ ands_openshift_hostname | default(ands_openshift_default_hostname) }}" + ands_openshift_public_cidr: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id), ands_openshift_default_cidr) }}" ands_openshift_public_ip: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id) | ipaddr('address'), ands_openshift_default_ip) }}" ands_openshift_public_hostname: "{{ ands_openshift_public_hostname | default(ands_openshift_default_hostname) }}" + ands_storage_cidr: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) }}" ands_storage_ip: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) | ipaddr('address') }}" ands_hostname_storage: "ands_storage{{ ands_host_id }}" ands_hostname_openshift: "ands_openshift{{ ands_host_id }}" @@ -33,27 +36,37 @@ - name: "Detect inner network interface" include_tasks: "find_interface_by_ip.yml" vars: - var: "ands_openshift_inner_interface" + var: "ands_inner_interface" ip: "{{ ands_openshift_ip }}" when: - - ands_openshift_inner_interface is not defined + - ands_inner_interface is not defined -- name: "Detect public network interface" +- name: "Detect public network interface (using public ip)" include_tasks: "find_interface_by_ip.yml" vars: - var: "ands_openshift_public_interface" + var: "ands_public_interface" ip: "{{ (ands_openshift_public_ip == ands_none) | ternary(ands_default_ip, ands_openshift_public_ip) }}" when: - - ands_openshift_public_interface is not defined + - ands_public_interface is not defined -- name: Set ipfailover interface - set_fact: - ands_ipfailover_interface: "{{ ands_openshift_public_interface }}" - when: ands_ipfailover_interface is not defined +- name: "Detect public network interface (using global network)" + include_tasks: "find_interface_by_net.yml" + vars: + var: "ands_public_interface" + net: "{{ ands_global_network }}" + when: + - ands_public_interface is not defined + - ands_global_network is defined - name: Set ipfailover inner interface set_fact: - ands_ipfailover_inner_interface: "{{ ands_openshift_inner_interface }}" + ands_ipfailover_inner_interface: "{{ ands_inner_interface }}" when: ands_ipfailover_inner_interface is not defined +- name: Set ipfailover interface + set_fact: + ands_ipfailover_interface: "{{ ands_public_interface }}" + when: ands_ipfailover_interface is not defined + + #- debug: msg="{{ hostvars }}" diff --git a/roles/ands_network/defaults/main.yml b/roles/ands_network/defaults/main.yml new file mode 100644 index 0000000..139e8b3 --- /dev/null +++ b/roles/ands_network/defaults/main.yml @@ -0,0 +1 @@ +configure_network: "{{ ands_configure_network | default(false) }}" diff --git a/roles/ands_network/tasks/nm_configure.yml b/roles/ands_network/tasks/nm_configure.yml new file mode 100644 index 0000000..4482705 --- /dev/null +++ b/roles/ands_network/tasks/nm_configure.yml @@ -0,0 +1,39 @@ + +- name: install needed network manager libs + yum: name='{{ item }}' state=installed + with_items: + - NetworkManager-glib + - nm-connection-editor + - libsemanage-python + - policycoreutils-python + +# Works in 2.4.3.0 with couple of upstream patches +# Infiniband is not supported yet +#- name: configure storage network interface +# nmcli: type=inifinband conn_name=storage ifname="{{ ands_storage_interface }}" ip4="{{ ands_storage_cidr }}" state="present" autoconnect="yes" + + +- name: configure storage nework + include_tasks: nm_configure_connection.yml + vars: + name: "storage" + iface: "{{ ands_storage_interface }}" + cidr: "{{ ands_storage_cidr }}" + force: true + +- name: configure openshift nework + include_tasks: nm_configure_connection.yml + vars: + name: "openshift" + iface: "{{ ands_inner_interface }}" + cidr: "{{ ands_openshift_cidr }}" + force: true + +- name: configure public nework + include_tasks: nm_configure_connection.yml + vars: + name: "storage" + iface: "{{ ands_public_interface }}" + cidr: "{{ ands_openshift_public_cidr }}" + alias: true + diff --git a/roles/ands_network/tasks/nm_configure_connection.yml b/roles/ands_network/tasks/nm_configure_connection.yml new file mode 100644 index 0000000..18fc91e --- /dev/null +++ b/roles/ands_network/tasks/nm_configure_connection.yml @@ -0,0 +1,51 @@ +- name: "detect nm connection corresponding to interface '{{ iface }}'" + shell: "nmcli d show {{ iface | quote }} | grep CONNECTION | cut -d ':' -f 2- | sed -E -e 's/^[[:space:]]+//' | grep '^[[:alpha:]]'" + register: conres + failed_when: false + changed_when: false + + +- name: "check if the requested ip '{{ cidr }}' is present on the interface '{{ iface }}'" + set_fact: + ip_present: "{{ cidr | ipaddr('address') in ips }}" + vars: + eth: "{{ hostvars[inventory_hostname]['ansible_' + iface] | default({}) }}" + ipv4: "{{ eth['ipv4'] | default({}) }}" + q: "{{ eth | json_query('ipv4_secondaries[*].address') }}" + sec: "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}" + ips: "{{ sec | union([ipv4.address]) }}" + when: + - conres.rc == 0 +# - eth.ipv4 is defined + +- name: "destroy connection '{{ conres.stdout }}' if ip does not match" + command: "nmcli connection delete {{ conres.stdout | quote }}" + register: delres + when: + - conres.rc == 0 + - force | default(false) + - not (alias | default(false)) + - not ip_present + +- name: "configure storage network interface '{{ iface }}' to '{{ cidr }}'" + command: "nmcli connection add type infiniband ifname {{ iface | quote }} con-name {{ name }} ip4 {{ cidr }}" + when: + - (conres.rc != 0) or (not (delres | skipped)) + - (conres.rc != 0) or (not (alias | default(false))) + +- name: "add ip alias '{{ cidr }}' to connection '{{ conres.stdout }}' using interface '{{ iface }}'" + command: "nmcli connection modify {{ conres.stdout | quote }} +ipv4.address {{ cidr }}" + register: alres + when: + - alias | default(false) + - conres.rc == 0 + - not ip_present + + +- name: "add ip alias '{{ cidr }}' to network interface '{{ iface }}'" + command: "nmcli connection up {{ conres.stdout | quote }}" + register: alres + when: + - not(alres | skipped) + - alres | succeeded + - not ip_present diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 30b1ff8..def846d 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -1,5 +1,7 @@ docker_exclude_vgs: "{{ ands_data_vg is defined | ternary( [ ands_data_vg ], [] ) }}" docker_lv: "docker-pool" +docker_root_lv: "docker-root-lv" +docker_setup_root: "{{ docker_root_volume_size is defined }}" docker_min_size: 100 docker_max_log_size: "2m" diff --git a/roles/docker/tasks/configure.yml b/roles/docker/tasks/configure.yml new file mode 100644 index 0000000..5d29291 --- /dev/null +++ b/roles/docker/tasks/configure.yml @@ -0,0 +1,63 @@ +--- +#- name: Remove docker +# yum: name="{{ item }}" state="absent" +# with_items: [ docker, docker-client, docker-common ] + +- name: install docker + include_tasks: install.yml + +- name: start docker + service: name="docker" state="started" + +- name: Configure bridge-nf-call-iptables with sysctl + sysctl: name="net.bridge.bridge-nf-call-iptables" value=1 state=present sysctl_set=yes + +- name: Configure bridge-nf-call-ip6tables with sysctl + sysctl: name="net.bridge.bridge-nf-call-ip6tables" value=1 state=present sysctl_set=yes + +- name: Determine if loopback + shell: docker info | grep 'Data file:.*loop' + register: loop_device_check + failed_when: false + changed_when: loop_device_check.rc == 0 + +- set_fact: docker_reinit="{{ (loop_device_check.rc == 0) or (vg == '') or (docker_setup_root and ((root_vg == '') or (vg != root_vg))) or (docker_storage_vg is defined and (docker_storage_vg != vg)) }}" + vars: + lv: "{{ ansible_lvm['lvs'][docker_lv] | default({}) }}" + vg: "{{ lv['vg'] | default('') }}" + root_lv: "{{ ansible_lvm['lvs'][docker_root_lv] | default({}) }}" + root_vg: "{{ root_lv['vg'] | default('') }}" + +- debug: msg="Re-initializing - {{ docker_reinit }}, Loopback check - {{ loop_device_check.stderr }}" + when: loop_device_check.stderr + +- import_tasks: storage.yml + when: docker_reinit + +- name: extend vg + lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="{{ docker_lv }}" size="{{ docker_volume_size }}" + when: + - not docker_reinit + - docker_volume_size is defined + +- name: extend root vg + lvol: vg="{{ ansible_lvm['lvs'][docker_root_lv]['vg'] }}" lv="{{ docker_root_lv }}" size="{{ docker_root_volume_size }}" + when: + - not docker_reinit + - docker_setup_root + - docker_root_volume_size is defined + +# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage +# The parameters to docker can be set trough OpenShift and currently are moved there. +#- name: Limit size of container log files +# ghetto_json: +# path: "/etc/docker/daemon.json" +# log-driver: "json-file" +# log-opts.max-size: "{{ docker_max_log_size }}" +# log-opts.max-file: "{{ docker_max_log_files }}" +# notify: +# - restart docker + +- name: start docker + service: name="docker" enabled=yes state=started + \ No newline at end of file diff --git a/roles/docker/tasks/install.yml b/roles/docker/tasks/install.yml new file mode 100644 index 0000000..41ae5a3 --- /dev/null +++ b/roles/docker/tasks/install.yml @@ -0,0 +1,13 @@ +- name: Remove versionlock from yum + command: yum versionlock delete docker docker-common docker-client + register: result + when: update | default(false) + failed_when: false + changed_when: result | failed + +- name: Ensure docker is installed + yum: name="docker{{ docker_version | default('') }}" state="{{ ((docker_version is defined) and (update | default(false))) | ternary('latest', 'present') }}" + +- name: Add versionlock to yum + command: yum versionlock add docker docker-common docker-client + when: docker_version is defined diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index c03d897..c9d0251 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,57 +1,3 @@ --- -#- name: Remove docker -# yum: name="{{ item }}" state="absent" -# with_items: [ docker, docker-client, docker-common ] - -- name: Remove versionlock from yum - command: yum versionlock delete docker docker-common docker-client - register: result - failed_when: false - changed_when: result | failed - -- name: Ensure docker is installed - yum: name="docker{{ docker_version | default('') }}" state="{{ docker_version is defined | ternary('latest', 'present') }}" - -- name: Add versionlock to yum - command: yum versionlock add docker docker-common docker-client - when: docker_version is defined - -- name: start docker - service: name="docker" state="started" - -- name: Configure bridge-nf-call-iptables with sysctl - sysctl: name="net.bridge.bridge-nf-call-iptables" value=1 state=present sysctl_set=yes - -- name: Configure bridge-nf-call-ip6tables with sysctl - sysctl: name="net.bridge.bridge-nf-call-ip6tables" value=1 state=present sysctl_set=yes - -- name: Determine if loopback - shell: docker info | grep 'Data file:.*loop' - register: loop_device_check - failed_when: false - changed_when: loop_device_check.rc == 0 - -- debug: msg="{{ loop_device_check.stderr }}" - when: loop_device_check.stderr - -- import_tasks: storage.yml - when: loop_device_check.rc == 0 - -- name: extend the vg - lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="docker_lv" size="{{ docker_volume_size }}" - when: docker_volume_size is defined - -# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage -# The parameters to docker can be set trough OpenShift and currently are moved there. -#- name: Limit size of container log files -# ghetto_json: -# path: "/etc/docker/daemon.json" -# log-driver: "json-file" -# log-opts.max-size: "{{ docker_max_log_size }}" -# log-opts.max-file: "{{ docker_max_log_files }}" -# notify: -# - restart docker - -- name: start docker - service: name="docker" enabled=yes state=started - \ No newline at end of file +- name: "Configuring network" + include_tasks: "{{ action | default('configure') }}.yml" diff --git a/roles/docker/tasks/storage.yml b/roles/docker/tasks/storage.yml index 5a5c858..e431030 100644 --- a/roles/docker/tasks/storage.yml +++ b/roles/docker/tasks/storage.yml @@ -1,11 +1,13 @@ --- +# We need to adjust docker_min_size if docker_volume_size set (min_size is just given in GB, but volume size +# support suffixes and we need to handle this). - name: list volume groups set_fact: available_vg="{{ item.key }}" with_dict: "{{ ansible_lvm.vgs }}" when: - not available_vg is defined - not item.key in ( docker_exclude_vgs | default([]) ) - - (item.value.free_g | int) > (docker_volume_size | default(docker_min_size)) + - (item.value.free_g | int) > docker_min_size - set_fact: docker_storage_vg="{{ available_vg }}" when: (not docker_storage_vg is defined) and (available_vg is defined) @@ -13,11 +15,13 @@ - fail: msg="Can't detect Docker VG" when: not docker_storage_vg is defined -- name: check to see if {{ docker_storage_device }} exists +- name: "check to see if {{ docker_storage_device }} exists" command: "test -e {{ docker_storage_device }}" when: docker_storage_device is defined -- set_fact: docker_storage_config="VG={{ docker_storage_vg }} AUTO_EXTEND_POOL=true" +- set_fact: docker_storage_config="VG={{ docker_storage_vg }} DATA_SIZE={{ docker_volume_size | default(docker_min_size ~ 'G') }} AUTO_EXTEND_POOL=true" +- set_fact: docker_storage_config="{{ docker_storage_config }} DOCKER_ROOT_VOLUME=yes DOCKER_ROOT_VOLUME_SIZE={{ docker_root_volume_size | default(docker_min_size ~ 'G') }}" + when: docker_setup_root - set_fact: docker_storage_config="{{ docker_storage_config }} DEVS={{ docker_storage_device }}" when: ( docker_storage_device is defined ) and ( ansible_lvm.vgs[docker_storage_vg] is not defined ) @@ -28,6 +32,29 @@ - name: delete /var/lib/docker file: path="/var/lib/docker" state=absent +- name: "delete {{ docker_lv }} lv" + lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="{{ docker_lv }}" state="absent" force="yes" + when: ansible_lvm['lvs'][docker_lv] is defined + +- name: delete docker-pool lv + lvol: vg="{{ ansible_lvm['lvs']['docker-pool']['vg'] }}" lv="docker-pool" state="absent" force="yes" + when: ansible_lvm['lvs']['docker-pool'] is defined + +- name: unmount docker partition + mount: path="/var/lib/docker" state="unmounted" + +- name: "delete {{ docker_root_lv }} lv" + lvol: vg="{{ ansible_lvm['lvs'][docker_root_lv]['vg'] }}" lv="{{ docker_root_lv }}" state="absent" force="yes" + when: ansible_lvm['lvs'][docker_root_lv] is defined + +- name: delete docker-pool lv + lvol: vg="{{ ansible_lvm['lvs']['docker-root-lv']['vg'] }}" lv="docker-root-lv" state="absent" force="yes" + when: ansible_lvm['lvs']['docker-root-lv'] is defined + + +- name: remove old configuration + file: path="/etc/sysconfig/docker-storage" state="absent" + - name: generate docker-storage-setup config file copy: content: "{{ docker_storage_config }}" @@ -38,4 +65,3 @@ - name: docker storage setup command: docker-storage-setup - diff --git a/roles/ofed/tasks/main.yml b/roles/ofed/tasks/main.yml new file mode 100644 index 0000000..bd85d43 --- /dev/null +++ b/roles/ofed/tasks/main.yml @@ -0,0 +1,6 @@ +- name: install the 'Infiniband support' package group + yum: name="@Infiniband Support" state=present + +- name: start rdma service + service: name="rdma" enabled=yes state=started + \ No newline at end of file -- cgit v1.2.1