From 006f333828db373435daa15483d2ab753048f62a Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Tue, 13 Mar 2018 05:01:30 +0100 Subject: Production --- group_vars/baremetal.yml | 17 +++++ group_vars/production.yml | 24 ++++++- inventories/production.erb | 74 ++++++++++++++++++++++ inventories/staging.erb | 2 + inventories/testing.erb | 2 + opts.sh | 1 + playbooks/ands-network.yml | 19 ++++++ playbooks/ands-prepare.yml | 5 -- playbooks/openshift-add-masters.yml | 3 +- playbooks/openshift-add-nodes.yml | 3 +- playbooks/x-os-update.yml | 9 +++ roles/ands_facts/defaults/main.yml | 6 +- roles/ands_facts/tasks/find_interface_by_net.yml | 17 +++++ roles/ands_facts/tasks/network.yml | 33 +++++++--- roles/ands_network/defaults/main.yml | 1 + roles/ands_network/tasks/nm_configure.yml | 39 ++++++++++++ .../ands_network/tasks/nm_configure_connection.yml | 51 +++++++++++++++ roles/docker/defaults/main.yml | 2 + roles/docker/tasks/configure.yml | 63 ++++++++++++++++++ roles/docker/tasks/install.yml | 13 ++++ roles/docker/tasks/main.yml | 58 +---------------- roles/docker/tasks/storage.yml | 34 ++++++++-- roles/ofed/tasks/main.yml | 6 ++ setup.sh | 6 ++ setup/users/htpasswd | 2 + 25 files changed, 409 insertions(+), 81 deletions(-) create mode 100644 inventories/production.erb create mode 100644 playbooks/ands-network.yml create mode 100644 playbooks/x-os-update.yml create mode 100644 roles/ands_facts/tasks/find_interface_by_net.yml create mode 100644 roles/ands_network/defaults/main.yml create mode 100644 roles/ands_network/tasks/nm_configure.yml create mode 100644 roles/ands_network/tasks/nm_configure_connection.yml create mode 100644 roles/docker/tasks/configure.yml create mode 100644 roles/docker/tasks/install.yml create mode 100644 roles/ofed/tasks/main.yml diff --git a/group_vars/baremetal.yml b/group_vars/baremetal.yml index 032e53f..640a00a 100644 --- a/group_vars/baremetal.yml +++ b/group_vars/baremetal.yml @@ -1,3 +1,20 @@ glusterfs_transport: rdma +# Size in GB to detect main Ands device +ands_data_device_threshold: 8192 + +# Primary LVM devices +ands_data_volume_size: "40T" +ands_heketi_volume_size: "4T" +docker_volume_size: "1T" +docker_root_volume_size: "1T" +docker_storage_vg: "ands" + +#Heketi Volumes +ands_registry_volume_size: "2T" + +ands_storage_interface: "ib0" +ands_inner_interface: "ib1" +#ands_public_interface: "eth0" + ands_host_id: "{{ ansible_hostname | regex_replace('^ipekatrin(\\d+)(\\.|$)', '\\1') }}" diff --git a/group_vars/production.yml b/group_vars/production.yml index 24b189f..c731873 100644 --- a/group_vars/production.yml +++ b/group_vars/production.yml @@ -1,7 +1,27 @@ +ands_configure_network: true ands_storage_network: 192.168.12.0/24 +ands_cluster_domain: ipe.kit.edu +ands_hostname_template: ipekatrin ands_openshift_lb: kaas.kit.edu ands_openshift_subdomain: kaas.kit.edu -ands_openshift_network: 192.168.26.0/24 +ands_openshift_network: 192.168.13.0/24 ands_openshift_public_network: 192.168.26.0/24 -ands_openshift_ingress_network: 192.168.16.0/22 +ands_openshift_ingress_network: 192.168.16.0/24 +ands_global_network: 141.52.64.0/23 + +#ands_inner_domain: "" +ands_openshift_set_hostname: false +# if we provision inner_lb (default), we can turn it on and just re-run ands_network role (or maintain play) +ands_use_inner_lb: false + + +#ands_ipfailover_interface: eth1 +ands_ipfailover_vips: [141.52.64.15/23, 141.52.64.17/23] + +katrin_openvpn_subnet_bits: 24 +katrin_openvpn_subnet_offset: 111 +katrin_openvpn_network: "192.168.0.0/16" + + +ands_prefer_docker: true diff --git a/inventories/production.erb b/inventories/production.erb new file mode 100644 index 0000000..575a86f --- /dev/null +++ b/inventories/production.erb @@ -0,0 +1,74 @@ +[masters] +ipekatrin[1:2].ipe.kit.edu + +[etcd] +ipekatrin[1:3].ipe.kit.edu + +[simple_storage_nodes] +ipekatrin[3:3].ipe.kit.edu + +[simple_nodes] + +[external_storage_servers] + +[production:children] +nodes +new_nodes +etcd +new_etcd +external_storage_servers + +[baremetal:children] +nodes +new_nodes +etcd +new_etcd +external_storage_servers + +[virtual] + +[OSEv3:children] +masters +new_masters +nodes +new_nodes +etcd +new_etcd + +[nodes:children] +masters +simple_storage_nodes +simple_nodes + +[storage_nodes:children] +masters +new_masters +simple_storage_nodes +new_simple_storage_nodes + +[glusterfs:children] +storage_nodes + +[ands_servers:children] +nodes +new_nodes +etcd +new_etcd +external_storage_servers + +[ands_storage_servers:children] +storage_nodes +external_storage_servers + +[ands:children] +ands_servers + +[new_masters] +[new_etcd] +[new_simple_storage_nodes] +[new_simple_nodes] + +[new_nodes:children] +new_masters +new_simple_storage_nodes +new_simple_nodes diff --git a/inventories/staging.erb b/inventories/staging.erb index aa9e935..3ce8a39 100644 --- a/inventories/staging.erb +++ b/inventories/staging.erb @@ -19,6 +19,8 @@ new_etcd external_storage_servers vagrant +[baremetal] + [virtual:children] nodes new_nodes diff --git a/inventories/testing.erb b/inventories/testing.erb index f9d27ae..9ee38ec 100644 --- a/inventories/testing.erb +++ b/inventories/testing.erb @@ -20,6 +20,8 @@ new_etcd external_storage_servers vagrant +[baremetal] + [virtual:children] nodes new_nodes diff --git a/opts.sh b/opts.sh index 9cfaf86..b17754e 100644 --- a/opts.sh +++ b/opts.sh @@ -17,6 +17,7 @@ Actions: Initial system installation all - full install, all the following steps in sequence: local - configure local ssh client + network - configure network interfaces prepare - perform all required pre-configuration before setting openshift openshift - setup OpenShift cluster gluster - configure gluster software and volumes diff --git a/playbooks/ands-network.yml b/playbooks/ands-network.yml new file mode 100644 index 0000000..1788781 --- /dev/null +++ b/playbooks/ands-network.yml @@ -0,0 +1,19 @@ +- name: Determin Ands facts + hosts: ands + roles: + - role: ands_facts + +- name: Prepare virtual hosts + hosts: virtual + roles: + - role: ands_vagrant_vmconf + +- name: Configure ininiband infrastrcture + hosts: baremetal + roles: + - role: ofed + +- name: Network configuration + hosts: ands + roles: + - { role: ands_network, action: nm_configure } diff --git a/playbooks/ands-prepare.yml b/playbooks/ands-prepare.yml index 239d292..4b204d2 100644 --- a/playbooks/ands-prepare.yml +++ b/playbooks/ands-prepare.yml @@ -3,11 +3,6 @@ roles: - role: ands_facts -- name: Prepare virtual hosts - hosts: virtual - roles: - - role: ands_vagrant_vmconf - - name: Common setup procedures hosts: ands roles: diff --git a/playbooks/openshift-add-masters.yml b/playbooks/openshift-add-masters.yml index 6878137..8c1672c 100644 --- a/playbooks/openshift-add-masters.yml +++ b/playbooks/openshift-add-masters.yml @@ -2,9 +2,10 @@ hosts: nodes:new_nodes roles: - { role: ands_facts } - - { role: ands_common, os_update: true } - { role: ands_network, action: install_pre } +- import_playbook: x-os-update.yml + # etcd will provisioned as well if node is listed in new_etcd - import_playbook: ../anslib/openshift-ansible/playbooks/prerequisites.yml - import_playbook: ../anslib/openshift-ansible/playbooks/openshift-master/scaleup.yml diff --git a/playbooks/openshift-add-nodes.yml b/playbooks/openshift-add-nodes.yml index 3d3efc4..6d3ace6 100644 --- a/playbooks/openshift-add-nodes.yml +++ b/playbooks/openshift-add-nodes.yml @@ -2,9 +2,10 @@ hosts: nodes:new_nodes roles: - { role: ands_facts } - - { role: ands_common, os_update: true } - { role: ands_network, action: install_pre } +- import_playbook: x-os-update.yml + # I am not sure if etcd will be automatic here. If not, we may need to run etcd scaleup afterwards # if node is also in new_etcd list - import_playbook: ../anslib/openshift-ansible/playbooks/prerequisites.yml diff --git a/playbooks/x-os-update.yml b/playbooks/x-os-update.yml new file mode 100644 index 0000000..b17ff20 --- /dev/null +++ b/playbooks/x-os-update.yml @@ -0,0 +1,9 @@ +- name: Configure cluster hosts names + hosts: nodes:new_nodes + roles: + - { role: ands_common, os_update: true } + +- name: Configure cluster hosts names + hosts: nodes:new_nodes + roles: + - { role: docker, action: install, update: true } diff --git a/roles/ands_facts/defaults/main.yml b/roles/ands_facts/defaults/main.yml index c74984e..5feb354 100644 --- a/roles/ands_facts/defaults/main.yml +++ b/roles/ands_facts/defaults/main.yml @@ -25,7 +25,9 @@ ands_cluster_domain: "{{ ansible_domain }}" ands_inner_domain: "{{ ands_cluster_domain }}" ands_default_ip: "{{ ansible_default_ipv4.address }}" +ands_default_cidr: "{{ ansible_default_ipv4.address ~ '/' ~ ( ansible_default_ipv4.netmask | ipaddr('prefix') ) }}" ands_openshift_default_ip: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_ip, ands_none) }}" +ands_openshift_default_cidr: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_cidr, ands_none) }}" ands_openshift_default_hostname: "{{ (ands_hostname_template is defined) | ternary(ands_hostname_template ~ ands_host_id, ansible_hostname) }}" ands_inner_lb: true @@ -33,5 +35,5 @@ ands_use_inner_lb: false ands_inner_lb_id: 254 ands_inner_lb_hostname: 'ands-lb' -#ands_openshift_inner_interface: -#ands_openshift_public_interface: +#ands_inner_interface: +#ands_public_interface: diff --git a/roles/ands_facts/tasks/find_interface_by_net.yml b/roles/ands_facts/tasks/find_interface_by_net.yml new file mode 100644 index 0000000..ad44578 --- /dev/null +++ b/roles/ands_facts/tasks/find_interface_by_net.yml @@ -0,0 +1,17 @@ +- name: "Looking for interface holding {{ net }}" + set_fact: + "{{ var }}": "{{ eth['device'] }}" + vars: + eth: "{{ hostvars[inventory_hostname]['ansible_' + item] | default({}) }}" + ipv4: "{{ eth['ipv4'] | default({}) }}" + q: "{{ eth | json_query('ipv4_secondaries[*].network') }}" + sec: "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}" + nets: "{{ sec | union([ipv4.network]) }}" + when: + - eth['type'] is defined + - eth['ipv4'] is defined + - eth['device'] is defined + - eth['type'] == 'ether' + - net | ipaddr('network') in nets + with_items: + - "{{ hostvars[inventory_hostname]['ansible_interfaces'] }}" diff --git a/roles/ands_facts/tasks/network.yml b/roles/ands_facts/tasks/network.yml index 808d7b6..1acafc1 100644 --- a/roles/ands_facts/tasks/network.yml +++ b/roles/ands_facts/tasks/network.yml @@ -12,10 +12,13 @@ ands_use_inner_lb: "{{ ands_use_inner_lb }}" ands_inner_lb_ip: "{{ ands_openshift_network | ipaddr(ands_inner_lb_id) | ipaddr('address') }}" ands_inner_lb_hostname: "{{ ands_inner_lb_hostname }}" + ands_openshift_cidr: "{{ ands_openshift_network | ipaddr(ands_host_id) }}" ands_openshift_ip: "{{ ands_openshift_network | ipaddr(ands_host_id) | ipaddr('address') }}" ands_openshift_hostname: "{{ ands_openshift_hostname | default(ands_openshift_default_hostname) }}" + ands_openshift_public_cidr: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id), ands_openshift_default_cidr) }}" ands_openshift_public_ip: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id) | ipaddr('address'), ands_openshift_default_ip) }}" ands_openshift_public_hostname: "{{ ands_openshift_public_hostname | default(ands_openshift_default_hostname) }}" + ands_storage_cidr: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) }}" ands_storage_ip: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) | ipaddr('address') }}" ands_hostname_storage: "ands_storage{{ ands_host_id }}" ands_hostname_openshift: "ands_openshift{{ ands_host_id }}" @@ -33,27 +36,37 @@ - name: "Detect inner network interface" include_tasks: "find_interface_by_ip.yml" vars: - var: "ands_openshift_inner_interface" + var: "ands_inner_interface" ip: "{{ ands_openshift_ip }}" when: - - ands_openshift_inner_interface is not defined + - ands_inner_interface is not defined -- name: "Detect public network interface" +- name: "Detect public network interface (using public ip)" include_tasks: "find_interface_by_ip.yml" vars: - var: "ands_openshift_public_interface" + var: "ands_public_interface" ip: "{{ (ands_openshift_public_ip == ands_none) | ternary(ands_default_ip, ands_openshift_public_ip) }}" when: - - ands_openshift_public_interface is not defined + - ands_public_interface is not defined -- name: Set ipfailover interface - set_fact: - ands_ipfailover_interface: "{{ ands_openshift_public_interface }}" - when: ands_ipfailover_interface is not defined +- name: "Detect public network interface (using global network)" + include_tasks: "find_interface_by_net.yml" + vars: + var: "ands_public_interface" + net: "{{ ands_global_network }}" + when: + - ands_public_interface is not defined + - ands_global_network is defined - name: Set ipfailover inner interface set_fact: - ands_ipfailover_inner_interface: "{{ ands_openshift_inner_interface }}" + ands_ipfailover_inner_interface: "{{ ands_inner_interface }}" when: ands_ipfailover_inner_interface is not defined +- name: Set ipfailover interface + set_fact: + ands_ipfailover_interface: "{{ ands_public_interface }}" + when: ands_ipfailover_interface is not defined + + #- debug: msg="{{ hostvars }}" diff --git a/roles/ands_network/defaults/main.yml b/roles/ands_network/defaults/main.yml new file mode 100644 index 0000000..139e8b3 --- /dev/null +++ b/roles/ands_network/defaults/main.yml @@ -0,0 +1 @@ +configure_network: "{{ ands_configure_network | default(false) }}" diff --git a/roles/ands_network/tasks/nm_configure.yml b/roles/ands_network/tasks/nm_configure.yml new file mode 100644 index 0000000..4482705 --- /dev/null +++ b/roles/ands_network/tasks/nm_configure.yml @@ -0,0 +1,39 @@ + +- name: install needed network manager libs + yum: name='{{ item }}' state=installed + with_items: + - NetworkManager-glib + - nm-connection-editor + - libsemanage-python + - policycoreutils-python + +# Works in 2.4.3.0 with couple of upstream patches +# Infiniband is not supported yet +#- name: configure storage network interface +# nmcli: type=inifinband conn_name=storage ifname="{{ ands_storage_interface }}" ip4="{{ ands_storage_cidr }}" state="present" autoconnect="yes" + + +- name: configure storage nework + include_tasks: nm_configure_connection.yml + vars: + name: "storage" + iface: "{{ ands_storage_interface }}" + cidr: "{{ ands_storage_cidr }}" + force: true + +- name: configure openshift nework + include_tasks: nm_configure_connection.yml + vars: + name: "openshift" + iface: "{{ ands_inner_interface }}" + cidr: "{{ ands_openshift_cidr }}" + force: true + +- name: configure public nework + include_tasks: nm_configure_connection.yml + vars: + name: "storage" + iface: "{{ ands_public_interface }}" + cidr: "{{ ands_openshift_public_cidr }}" + alias: true + diff --git a/roles/ands_network/tasks/nm_configure_connection.yml b/roles/ands_network/tasks/nm_configure_connection.yml new file mode 100644 index 0000000..18fc91e --- /dev/null +++ b/roles/ands_network/tasks/nm_configure_connection.yml @@ -0,0 +1,51 @@ +- name: "detect nm connection corresponding to interface '{{ iface }}'" + shell: "nmcli d show {{ iface | quote }} | grep CONNECTION | cut -d ':' -f 2- | sed -E -e 's/^[[:space:]]+//' | grep '^[[:alpha:]]'" + register: conres + failed_when: false + changed_when: false + + +- name: "check if the requested ip '{{ cidr }}' is present on the interface '{{ iface }}'" + set_fact: + ip_present: "{{ cidr | ipaddr('address') in ips }}" + vars: + eth: "{{ hostvars[inventory_hostname]['ansible_' + iface] | default({}) }}" + ipv4: "{{ eth['ipv4'] | default({}) }}" + q: "{{ eth | json_query('ipv4_secondaries[*].address') }}" + sec: "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}" + ips: "{{ sec | union([ipv4.address]) }}" + when: + - conres.rc == 0 +# - eth.ipv4 is defined + +- name: "destroy connection '{{ conres.stdout }}' if ip does not match" + command: "nmcli connection delete {{ conres.stdout | quote }}" + register: delres + when: + - conres.rc == 0 + - force | default(false) + - not (alias | default(false)) + - not ip_present + +- name: "configure storage network interface '{{ iface }}' to '{{ cidr }}'" + command: "nmcli connection add type infiniband ifname {{ iface | quote }} con-name {{ name }} ip4 {{ cidr }}" + when: + - (conres.rc != 0) or (not (delres | skipped)) + - (conres.rc != 0) or (not (alias | default(false))) + +- name: "add ip alias '{{ cidr }}' to connection '{{ conres.stdout }}' using interface '{{ iface }}'" + command: "nmcli connection modify {{ conres.stdout | quote }} +ipv4.address {{ cidr }}" + register: alres + when: + - alias | default(false) + - conres.rc == 0 + - not ip_present + + +- name: "add ip alias '{{ cidr }}' to network interface '{{ iface }}'" + command: "nmcli connection up {{ conres.stdout | quote }}" + register: alres + when: + - not(alres | skipped) + - alres | succeeded + - not ip_present diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 30b1ff8..def846d 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -1,5 +1,7 @@ docker_exclude_vgs: "{{ ands_data_vg is defined | ternary( [ ands_data_vg ], [] ) }}" docker_lv: "docker-pool" +docker_root_lv: "docker-root-lv" +docker_setup_root: "{{ docker_root_volume_size is defined }}" docker_min_size: 100 docker_max_log_size: "2m" diff --git a/roles/docker/tasks/configure.yml b/roles/docker/tasks/configure.yml new file mode 100644 index 0000000..5d29291 --- /dev/null +++ b/roles/docker/tasks/configure.yml @@ -0,0 +1,63 @@ +--- +#- name: Remove docker +# yum: name="{{ item }}" state="absent" +# with_items: [ docker, docker-client, docker-common ] + +- name: install docker + include_tasks: install.yml + +- name: start docker + service: name="docker" state="started" + +- name: Configure bridge-nf-call-iptables with sysctl + sysctl: name="net.bridge.bridge-nf-call-iptables" value=1 state=present sysctl_set=yes + +- name: Configure bridge-nf-call-ip6tables with sysctl + sysctl: name="net.bridge.bridge-nf-call-ip6tables" value=1 state=present sysctl_set=yes + +- name: Determine if loopback + shell: docker info | grep 'Data file:.*loop' + register: loop_device_check + failed_when: false + changed_when: loop_device_check.rc == 0 + +- set_fact: docker_reinit="{{ (loop_device_check.rc == 0) or (vg == '') or (docker_setup_root and ((root_vg == '') or (vg != root_vg))) or (docker_storage_vg is defined and (docker_storage_vg != vg)) }}" + vars: + lv: "{{ ansible_lvm['lvs'][docker_lv] | default({}) }}" + vg: "{{ lv['vg'] | default('') }}" + root_lv: "{{ ansible_lvm['lvs'][docker_root_lv] | default({}) }}" + root_vg: "{{ root_lv['vg'] | default('') }}" + +- debug: msg="Re-initializing - {{ docker_reinit }}, Loopback check - {{ loop_device_check.stderr }}" + when: loop_device_check.stderr + +- import_tasks: storage.yml + when: docker_reinit + +- name: extend vg + lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="{{ docker_lv }}" size="{{ docker_volume_size }}" + when: + - not docker_reinit + - docker_volume_size is defined + +- name: extend root vg + lvol: vg="{{ ansible_lvm['lvs'][docker_root_lv]['vg'] }}" lv="{{ docker_root_lv }}" size="{{ docker_root_volume_size }}" + when: + - not docker_reinit + - docker_setup_root + - docker_root_volume_size is defined + +# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage +# The parameters to docker can be set trough OpenShift and currently are moved there. +#- name: Limit size of container log files +# ghetto_json: +# path: "/etc/docker/daemon.json" +# log-driver: "json-file" +# log-opts.max-size: "{{ docker_max_log_size }}" +# log-opts.max-file: "{{ docker_max_log_files }}" +# notify: +# - restart docker + +- name: start docker + service: name="docker" enabled=yes state=started + \ No newline at end of file diff --git a/roles/docker/tasks/install.yml b/roles/docker/tasks/install.yml new file mode 100644 index 0000000..41ae5a3 --- /dev/null +++ b/roles/docker/tasks/install.yml @@ -0,0 +1,13 @@ +- name: Remove versionlock from yum + command: yum versionlock delete docker docker-common docker-client + register: result + when: update | default(false) + failed_when: false + changed_when: result | failed + +- name: Ensure docker is installed + yum: name="docker{{ docker_version | default('') }}" state="{{ ((docker_version is defined) and (update | default(false))) | ternary('latest', 'present') }}" + +- name: Add versionlock to yum + command: yum versionlock add docker docker-common docker-client + when: docker_version is defined diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index c03d897..c9d0251 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,57 +1,3 @@ --- -#- name: Remove docker -# yum: name="{{ item }}" state="absent" -# with_items: [ docker, docker-client, docker-common ] - -- name: Remove versionlock from yum - command: yum versionlock delete docker docker-common docker-client - register: result - failed_when: false - changed_when: result | failed - -- name: Ensure docker is installed - yum: name="docker{{ docker_version | default('') }}" state="{{ docker_version is defined | ternary('latest', 'present') }}" - -- name: Add versionlock to yum - command: yum versionlock add docker docker-common docker-client - when: docker_version is defined - -- name: start docker - service: name="docker" state="started" - -- name: Configure bridge-nf-call-iptables with sysctl - sysctl: name="net.bridge.bridge-nf-call-iptables" value=1 state=present sysctl_set=yes - -- name: Configure bridge-nf-call-ip6tables with sysctl - sysctl: name="net.bridge.bridge-nf-call-ip6tables" value=1 state=present sysctl_set=yes - -- name: Determine if loopback - shell: docker info | grep 'Data file:.*loop' - register: loop_device_check - failed_when: false - changed_when: loop_device_check.rc == 0 - -- debug: msg="{{ loop_device_check.stderr }}" - when: loop_device_check.stderr - -- import_tasks: storage.yml - when: loop_device_check.rc == 0 - -- name: extend the vg - lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="docker_lv" size="{{ docker_volume_size }}" - when: docker_volume_size is defined - -# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage -# The parameters to docker can be set trough OpenShift and currently are moved there. -#- name: Limit size of container log files -# ghetto_json: -# path: "/etc/docker/daemon.json" -# log-driver: "json-file" -# log-opts.max-size: "{{ docker_max_log_size }}" -# log-opts.max-file: "{{ docker_max_log_files }}" -# notify: -# - restart docker - -- name: start docker - service: name="docker" enabled=yes state=started - \ No newline at end of file +- name: "Configuring network" + include_tasks: "{{ action | default('configure') }}.yml" diff --git a/roles/docker/tasks/storage.yml b/roles/docker/tasks/storage.yml index 5a5c858..e431030 100644 --- a/roles/docker/tasks/storage.yml +++ b/roles/docker/tasks/storage.yml @@ -1,11 +1,13 @@ --- +# We need to adjust docker_min_size if docker_volume_size set (min_size is just given in GB, but volume size +# support suffixes and we need to handle this). - name: list volume groups set_fact: available_vg="{{ item.key }}" with_dict: "{{ ansible_lvm.vgs }}" when: - not available_vg is defined - not item.key in ( docker_exclude_vgs | default([]) ) - - (item.value.free_g | int) > (docker_volume_size | default(docker_min_size)) + - (item.value.free_g | int) > docker_min_size - set_fact: docker_storage_vg="{{ available_vg }}" when: (not docker_storage_vg is defined) and (available_vg is defined) @@ -13,11 +15,13 @@ - fail: msg="Can't detect Docker VG" when: not docker_storage_vg is defined -- name: check to see if {{ docker_storage_device }} exists +- name: "check to see if {{ docker_storage_device }} exists" command: "test -e {{ docker_storage_device }}" when: docker_storage_device is defined -- set_fact: docker_storage_config="VG={{ docker_storage_vg }} AUTO_EXTEND_POOL=true" +- set_fact: docker_storage_config="VG={{ docker_storage_vg }} DATA_SIZE={{ docker_volume_size | default(docker_min_size ~ 'G') }} AUTO_EXTEND_POOL=true" +- set_fact: docker_storage_config="{{ docker_storage_config }} DOCKER_ROOT_VOLUME=yes DOCKER_ROOT_VOLUME_SIZE={{ docker_root_volume_size | default(docker_min_size ~ 'G') }}" + when: docker_setup_root - set_fact: docker_storage_config="{{ docker_storage_config }} DEVS={{ docker_storage_device }}" when: ( docker_storage_device is defined ) and ( ansible_lvm.vgs[docker_storage_vg] is not defined ) @@ -28,6 +32,29 @@ - name: delete /var/lib/docker file: path="/var/lib/docker" state=absent +- name: "delete {{ docker_lv }} lv" + lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="{{ docker_lv }}" state="absent" force="yes" + when: ansible_lvm['lvs'][docker_lv] is defined + +- name: delete docker-pool lv + lvol: vg="{{ ansible_lvm['lvs']['docker-pool']['vg'] }}" lv="docker-pool" state="absent" force="yes" + when: ansible_lvm['lvs']['docker-pool'] is defined + +- name: unmount docker partition + mount: path="/var/lib/docker" state="unmounted" + +- name: "delete {{ docker_root_lv }} lv" + lvol: vg="{{ ansible_lvm['lvs'][docker_root_lv]['vg'] }}" lv="{{ docker_root_lv }}" state="absent" force="yes" + when: ansible_lvm['lvs'][docker_root_lv] is defined + +- name: delete docker-pool lv + lvol: vg="{{ ansible_lvm['lvs']['docker-root-lv']['vg'] }}" lv="docker-root-lv" state="absent" force="yes" + when: ansible_lvm['lvs']['docker-root-lv'] is defined + + +- name: remove old configuration + file: path="/etc/sysconfig/docker-storage" state="absent" + - name: generate docker-storage-setup config file copy: content: "{{ docker_storage_config }}" @@ -38,4 +65,3 @@ - name: docker storage setup command: docker-storage-setup - diff --git a/roles/ofed/tasks/main.yml b/roles/ofed/tasks/main.yml new file mode 100644 index 0000000..bd85d43 --- /dev/null +++ b/roles/ofed/tasks/main.yml @@ -0,0 +1,6 @@ +- name: install the 'Infiniband support' package group + yum: name="@Infiniband Support" state=present + +- name: start rdma service + service: name="rdma" enabled=yes state=started + \ No newline at end of file diff --git a/setup.sh b/setup.sh index 4ccf94d..69293aa 100755 --- a/setup.sh +++ b/setup.sh @@ -7,6 +7,7 @@ shift case "$action" in all) + ./setup.sh -i $inventory network "$@" || exit 1 ./setup.sh -i $inventory prepare "$@" || exit 1 ./setup.sh -i $inventory openshift "$@" || exit 1 ./setup.sh -i $inventory gluster "$@" || exit 1 @@ -19,6 +20,9 @@ case "$action" in vm) apply playbooks/ands-vm-setup.yml "$@" || exit 1 ;; + network) + apply playbooks/ands-network.yml "$@" || exit 1 + ;; prepare) apply playbooks/ands-prepare.yml "$@" || exit 1 ;; @@ -64,12 +68,14 @@ case "$action" in apply playbooks/openshift-add-gluster.yml "$@" || exit 1 ;; masters) + ./setup.sh -i $inventory network "$@" || exit 1 ./setup.sh -i $inventory prepare "$@" || exit 1 ./setup.sh -i $inventory openshift-masters "$@" || exit 1 ./setup.sh -i $inventory gluster "$@" || exit 1 ./setup.sh -i $inventory configure "$@" || exit 1 ;; nodes) + ./setup.sh -i $inventory network "$@" || exit 1 ./setup.sh -i $inventory prepare "$@" || exit 1 ./setup.sh -i $inventory openshift-nodes "$@" || exit 1 ./setup.sh -i $inventory gluster "$@" || exit 1 diff --git a/setup/users/htpasswd b/setup/users/htpasswd index cf0d67e..cf0f1e6 100644 --- a/setup/users/htpasswd +++ b/setup/users/htpasswd @@ -1,3 +1,5 @@ pdv:$apr1$ACvj6uUa$Nm1Vq8hZq3RzTtaYpAHv01 csa:$apr1$IqEwdnzy$UAdd8ZSFnXommBbj29w3c0 katrin:$apr1$/hxgbxC4$/MxeHtIYvAJcIQFR5Jz0E0 +ntj:$apr1$un8GkxMv$VZ36KeB90qwMMVpvHxOj8. +kopmann:CZFk3ASLX0Vq6 -- cgit v1.2.1