GlusterFS playbook and role

Signed-off-by: Jose A. Rivera <jarrpa@redhat.com>

5 files changed, 487 insertions, 0 deletions

diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml
new file mode 100644
index 000000000..26ca5eebf
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml
@@ -0,0 +1,107 @@
+---
+- assert:
+    that: "openshift_storage_glusterfs_nodeselector.keys() | count == 1"
+    msg: Only one GlusterFS nodeselector key pair should be provided
+
+- assert:
+    that: "groups.oo_glusterfs_to_config | count >= 3"
+    msg: There must be at least three GlusterFS nodes specified
+
+- name: Delete pre-existing GlusterFS resources
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: "template,daemonset"
+    name: glusterfs
+    state: absent
+  when: openshift_storage_glusterfs_wipe
+
+- name: Unlabel any existing GlusterFS nodes
+  oc_label:
+    name: "{{ item }}"
+    kind: node
+    state: absent
+    labels: "{{ openshift_storage_glusterfs_nodeselector | oo_dict_to_list_of_dict }}"
+  with_items: "{{ groups.all }}"
+  when: openshift_storage_glusterfs_wipe
+
+- name: Delete pre-existing GlusterFS config
+  file:
+    path: /var/lib/glusterd
+    state: absent
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups.oo_glusterfs_to_config }}"
+  when: openshift_storage_glusterfs_wipe
+
+- name: Get GlusterFS storage devices state
+  command: "pvdisplay -C --noheadings -o pv_name,vg_name {% for device in hostvars[item].glusterfs_devices %}{{ device }} {% endfor %}"
+  register: devices_info
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups.oo_glusterfs_to_config }}"
+  failed_when: False
+  when: openshift_storage_glusterfs_wipe
+
+  # Runs "vgremove -fy <vg>; pvremove -fy <pv>" for every device found to be a physical volume.
+- name: Clear GlusterFS storage device contents
+  shell: "{% for line in item.stdout_lines %}{% set fields = line.split() %}{% if fields | count > 1 %}vgremove -fy {{ fields[1] }}; {% endif %}pvremove -fy {{ fields[0] }}; {% endfor %}"
+  delegate_to: "{{ item.item }}"
+  with_items: "{{ devices_info.results }}"
+  when:
+  - openshift_storage_glusterfs_wipe
+  - item.stdout_lines | count > 0
+
+- name: Add service accounts to privileged SCC
+  oc_adm_policy_user:
+    user: "system:serviceaccount:{{ openshift_storage_glusterfs_namespace }}:{{ item }}"
+    resource_kind: scc
+    resource_name: privileged
+    state: present
+  with_items:
+  - 'default'
+  - 'router'
+
+- name: Label GlusterFS nodes
+  oc_label:
+    name: "{{ glusterfs_host }}"
+    kind: node
+    state: add
+    labels: "{{ openshift_storage_glusterfs_nodeselector | oo_dict_to_list_of_dict }}"
+  with_items: "{{ groups.oo_glusterfs_to_config }}"
+  loop_control:
+    loop_var: glusterfs_host
+
+- name: Copy GlusterFS DaemonSet template
+  copy:
+    src: "{{ openshift.common.examples_content_version }}/glusterfs-template.yml"
+    dest: "{{ mktemp.stdout }}/glusterfs-template.yml"
+
+- name: Create GlusterFS template
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: template
+    name: glusterfs
+    state: present
+    files:
+    - "{{ mktemp.stdout }}/glusterfs-template.yml"
+
+- name: Deploy GlusterFS pods
+  oc_process:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    template_name: "glusterfs"
+    create: True
+    params:
+      IMAGE_NAME: "{{ openshift_storage_glusterfs_image }}"
+      IMAGE_VERSION: "{{ openshift_storage_glusterfs_version }}"
+
+- name: Wait for GlusterFS pods
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: pod
+    state: list
+    selector: "glusterfs-node=pod"
+  register: glusterfs_pods
+  until:
+  - "glusterfs_pods.results.results[0]['items'] | count > 0"
+  # There must be as many pods with 'Ready' staus  True as there are nodes expecting those pods
+  - "glusterfs_pods.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == groups.oo_glusterfs_to_config | count"
+  delay: 10
+  retries: "{{ (openshift_storage_glusterfs_timeout / 10) | int }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
new file mode 100644
index 000000000..9f092d5d5
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
@@ -0,0 +1,48 @@
+---
+- name: Delete pre-existing GlusterFS registry resources
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: "{{ item.kind }}"
+    name: "{{ item.name | default(omit) }}"
+    selector: "{{ item.selector | default(omit) }}"
+    state: absent
+  with_items:
+  - kind: "svc,ep"
+    name: "glusterfs-registry-endpoints"
+  failed_when: False
+
+- name: Generate GlusterFS registry endpoints
+  template:
+    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-endpoints.yml.j2"
+    dest: "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
+
+- name: Copy GlusterFS registry service
+  copy:
+    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml"
+    dest: "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
+
+- name: Create GlusterFS registry endpoints
+  oc_obj:
+    namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
+    state: present
+    kind: endpoints
+    name: glusterfs-registry-endpoints
+    files:
+    - "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
+
+- name: Create GlusterFS registry service
+  oc_obj:
+    namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
+    state: present
+    kind: service
+    name: glusterfs-registry-endpoints
+    files:
+    - "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
+
+- name: Check if GlusterFS registry volume exists
+  command: "heketi-cli -s http://{{ openshift_storage_glusterfs_heketi_url }} --user admin --secret '{{ openshift_storage_glusterfs_heketi_admin_key }}' volume list"
+  register: registry_volume
+
+- name: Create GlusterFS registry volume
+  command: "heketi-cli -s http://{{ openshift_storage_glusterfs_heketi_url }} --user admin --secret '{{ openshift_storage_glusterfs_heketi_admin_key }}' volume create --size={{ openshift.hosted.registry.storage.volume.size | replace('Gi','') }} --name={{ openshift.hosted.registry.storage.glusterfs.path }}"
+  when: "'{{ openshift.hosted.registry.storage.glusterfs.path }}' not in registry_volume.stdout"
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
new file mode 100644
index 000000000..76ae1db75
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
@@ -0,0 +1,41 @@
+---
+- name: Copy initial heketi resource files
+  copy:
+    src: "{{ openshift.common.examples_content_version }}/{{ item }}"
+    dest: "{{ mktemp.stdout }}/{{ item }}"
+  with_items:
+  - "deploy-heketi-template.yml"
+
+- name: Create deploy-heketi resources
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: template
+    name: deploy-heketi
+    state: present
+    files:
+    - "{{ mktemp.stdout }}/deploy-heketi-template.yml"
+
+- name: Deploy deploy-heketi pod
+  oc_process:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    template_name: "deploy-heketi"
+    create: True
+    params:
+      IMAGE_NAME: "{{ openshift_storage_glusterfs_heketi_image }}"
+      IMAGE_VERSION: "{{ openshift_storage_glusterfs_heketi_version }}"
+      HEKETI_USER_KEY: "{{ openshift_storage_glusterfs_heketi_user_key }}"
+      HEKETI_ADMIN_KEY: "{{ openshift_storage_glusterfs_heketi_admin_key }}"
+
+- name: Wait for deploy-heketi pod
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: pod
+    state: list
+    selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support"
+  register: heketi_pod
+  until:
+  - "heketi_pod.results.results[0]['items'] | count > 0"
+  # Pod's 'Ready' status must be True
+  - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1"
+  delay: 10
+  retries: "{{ (openshift_storage_glusterfs_timeout / 10) | int }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
new file mode 100644
index 000000000..84b85e95d
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
@@ -0,0 +1,109 @@
+---
+- name: Create heketi DB volume
+  command: "heketi-cli -s http://{{ openshift_storage_glusterfs_heketi_url }} --user admin --secret '{{ openshift_storage_glusterfs_heketi_admin_key }}' setup-openshift-heketi-storage --listfile {{ mktemp.stdout }}/heketi-storage.json"
+  register: setup_storage
+  failed_when: False
+
+# This is used in the subsequent task
+- name: Copy the admin client config
+  command: >
+    cp {{ openshift.common.config_base }}/master/admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig
+  changed_when: False
+  check_mode: no
+
+# Need `command` here because heketi-storage.json contains multiple objects.
+- name: Copy heketi DB to GlusterFS volume
+  command: "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create -f {{ mktemp.stdout }}/heketi-storage.json -n {{ openshift_storage_glusterfs_namespace }}"
+  when: "setup_storage.rc == 0"
+
+- name: Wait for copy job to finish
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: job
+    state: list
+    name: "heketi-storage-copy-job"
+  register: heketi_job
+  until:
+  - "'results' in heketi_job.results and heketi_job.results.results | count > 0"
+  # Pod's 'Complete' status must be True
+  - "heketi_job.results.results | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Complete'}) | map('bool') | select | list | count == 1"
+  delay: 10
+  retries: "{{ (openshift_storage_glusterfs_timeout / 10) | int }}"
+  failed_when:
+  - "'results' in heketi_job.results"
+  - "heketi_job.results.results | count > 0"
+  # Fail when pod's 'Failed' status is True
+  - "heketi_job.results.results | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Failed'}) | map('bool') | select | list | count == 1"
+  when: "setup_storage.rc == 0"
+
+- name: Delete deploy resources
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: "{{ item.kind }}"
+    name: "{{ item.name | default(omit) }}"
+    selector: "{{ item.selector | default(omit) }}"
+    state: absent
+  with_items:
+  - kind: "template,route,service,jobs,dc,secret"
+    selector: "deploy-heketi"
+  failed_when: False
+
+- name: Copy heketi template
+  copy:
+    src: "{{ openshift.common.examples_content_version }}/heketi-template.yml"
+    dest: "{{ mktemp.stdout }}/heketi-template.yml"
+
+- name: Create heketi resources
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: template
+    name: heketi
+    state: present
+    files:
+    - "{{ mktemp.stdout }}/heketi-template.yml"
+
+- name: Deploy heketi pod
+  oc_process:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    template_name: "heketi"
+    create: True
+    params:
+      IMAGE_NAME: "{{ openshift_storage_glusterfs_heketi_image }}"
+      IMAGE_VERSION: "{{ openshift_storage_glusterfs_heketi_version }}"
+      HEKETI_USER_KEY: "{{ openshift_storage_glusterfs_heketi_user_key }}"
+      HEKETI_ADMIN_KEY: "{{ openshift_storage_glusterfs_heketi_admin_key }}"
+
+- name: Wait for heketi pod
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: pod
+    state: list
+    selector: "glusterfs=heketi-pod"
+  register: heketi_pod
+  until:
+  - "heketi_pod.results.results[0]['items'] | count > 0"
+  # Pod's 'Ready' status must be True
+  - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1"
+  delay: 10
+  retries: "{{ (openshift_storage_glusterfs_timeout / 10) | int }}"
+
+- name: Determine heketi URL
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    state: list
+    kind: ep
+    selector: "glusterfs=heketi-service"
+  register: heketi_url
+  until:
+  - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''"
+  - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''"
+  delay: 10
+  retries: "{{ (openshift_storage_glusterfs_timeout / 10) | int }}"
+
+- name: Set heketi URL
+  set_fact:
+    openshift_storage_glusterfs_heketi_url: "{{ heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip }}:{{ heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port }}"
+
+- name: Verify heketi service
+  command: "heketi-cli -s http://{{ openshift_storage_glusterfs_heketi_url }} --user admin --secret '{{ openshift_storage_glusterfs_heketi_admin_key }}' cluster list"
+  changed_when: False
diff --git a/roles/openshift_storage_glusterfs/tasks/main.yml b/roles/openshift_storage_glusterfs/tasks/main.yml
new file mode 100644
index 000000000..265a3cc6e
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/tasks/main.yml
@@ -0,0 +1,182 @@
+---
+- name: Create temp directory for doing work in
+  command: mktemp -d /tmp/openshift-glusterfs-ansible-XXXXXX
+  register: mktemp
+  changed_when: False
+  check_mode: no
+
+- name: Verify target namespace exists
+  oc_project:
+    state: present
+    name: "{{ openshift_storage_glusterfs_namespace }}"
+  when: openshift_storage_glusterfs_is_native or openshift_storage_glusterfs_heketi_is_native
+
+- include: glusterfs_deploy.yml
+  when: openshift_storage_glusterfs_is_native
+
+- name: Make sure heketi-client is installed
+  package: name=heketi-client state=present
+
+- name: Delete pre-existing heketi resources
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: "{{ item.kind }}"
+    name: "{{ item.name | default(omit) }}"
+    selector: "{{ item.selector | default(omit) }}"
+    state: absent
+  with_items:
+  - kind: "template,route,service,jobs,dc,secret"
+    selector: "deploy-heketi"
+  - kind: "template,route,dc,service"
+    name: "heketi"
+  - kind: "svc,ep"
+    name: "heketi-storage-endpoints"
+  - kind: "sa"
+    name: "heketi-service-account"
+  failed_when: False
+  when: openshift_storage_glusterfs_heketi_wipe
+
+- name: Wait for deploy-heketi pods to terminate
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: pod
+    state: list
+    selector: "glusterfs=deploy-heketi-pod"
+  register: heketi_pod
+  until: "heketi_pod.results.results[0]['items'] | count == 0"
+  delay: 10
+  retries: "{{ (openshift_storage_glusterfs_timeout / 10) | int }}"
+  when: openshift_storage_glusterfs_heketi_wipe
+
+- name: Wait for heketi pods to terminate
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    kind: pod
+    state: list
+    selector: "glusterfs=heketi-pod"
+  register: heketi_pod
+  until: "heketi_pod.results.results[0]['items'] | count == 0"
+  delay: 10
+  retries: "{{ (openshift_storage_glusterfs_timeout / 10) | int }}"
+  when: openshift_storage_glusterfs_heketi_wipe
+
+- name: Create heketi service account
+  oc_serviceaccount:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    name: heketi-service-account
+    state: present
+  when: openshift_storage_glusterfs_heketi_is_native
+
+- name: Add heketi service account to privileged SCC
+  oc_adm_policy_user:
+    user: "system:serviceaccount:{{ openshift_storage_glusterfs_namespace }}:heketi-service-account"
+    resource_kind: scc
+    resource_name: privileged
+    state: present
+  when: openshift_storage_glusterfs_heketi_is_native
+
+- name: Allow heketi service account to view/edit pods
+  oc_adm_policy_user:
+    user: "system:serviceaccount:{{ openshift_storage_glusterfs_namespace }}:heketi-service-account"
+    resource_kind: role
+    resource_name: edit
+    state: present
+  when: openshift_storage_glusterfs_heketi_is_native
+
+- name: Check for existing deploy-heketi pod
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    state: list
+    kind: pod
+    selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support"
+  register: heketi_pod
+  when: openshift_storage_glusterfs_heketi_is_native
+
+- name: Check if need to deploy deploy-heketi
+  set_fact:
+    openshift_storage_glusterfs_heketi_deploy_is_missing: False
+  when:
+  - "openshift_storage_glusterfs_heketi_is_native"
+  - "heketi_pod.results.results[0]['items'] | count > 0"
+  # deploy-heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True
+  - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0"
+
+- name: Check for existing heketi pod
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    state: list
+    kind: pod
+    selector: "glusterfs=heketi-pod"
+  register: heketi_pod
+  when: openshift_storage_glusterfs_heketi_is_native
+
+- name: Check if need to deploy heketi
+  set_fact:
+    openshift_storage_glusterfs_heketi_is_missing: False
+  when:
+  - "openshift_storage_glusterfs_heketi_is_native"
+  - "heketi_pod.results.results[0]['items'] | count > 0"
+  # heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True
+  - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0"
+
+- include: heketi_deploy_part1.yml
+  when:
+  - openshift_storage_glusterfs_heketi_is_native
+  - openshift_storage_glusterfs_heketi_deploy_is_missing
+  - openshift_storage_glusterfs_heketi_is_missing
+
+- name: Determine heketi URL
+  oc_obj:
+    namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    state: list
+    kind: ep
+    selector: "glusterfs in (deploy-heketi-service, heketi-service)"
+  register: heketi_url
+  until:
+  - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''"
+  - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''"
+  delay: 10
+  retries: "{{ (openshift_storage_glusterfs_timeout / 10) | int }}"
+  when:
+  - openshift_storage_glusterfs_heketi_is_native
+  - openshift_storage_glusterfs_heketi_url is undefined
+
+- name: Set heketi URL
+  set_fact:
+    openshift_storage_glusterfs_heketi_url: "{{ heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip }}:{{ heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port }}"
+  when:
+  - openshift_storage_glusterfs_heketi_is_native
+  - openshift_storage_glusterfs_heketi_url is undefined
+
+- name: Verify heketi service
+  command: "heketi-cli -s http://{{ openshift_storage_glusterfs_heketi_url }} --user admin --secret '{{ openshift_storage_glusterfs_heketi_admin_key }}' cluster list"
+  changed_when: False
+
+- name: Generate topology file
+  template:
+    src: "{{ openshift.common.examples_content_version }}/topology.json.j2"
+    dest: "{{ mktemp.stdout }}/topology.json"
+  when:
+  - openshift_storage_glusterfs_is_native
+  - openshift_storage_glusterfs_heketi_topology_load
+
+- name: Load heketi topology
+  command: "heketi-cli -s http://{{ openshift_storage_glusterfs_heketi_url }} --user admin --secret '{{ openshift_storage_glusterfs_heketi_admin_key }}' topology load --json={{ mktemp.stdout }}/topology.json 2>&1"
+  register: topology_load
+  failed_when: "topology_load.rc != 0 or 'Unable' in topology_load.stdout"
+  when:
+  - openshift_storage_glusterfs_is_native
+  - openshift_storage_glusterfs_heketi_topology_load
+
+- include: heketi_deploy_part2.yml
+  when: openshift_storage_glusterfs_heketi_is_native and openshift_storage_glusterfs_heketi_is_missing
+
+- include: glusterfs_registry.yml
+  when: "openshift.hosted.registry.storage.kind == 'glusterfs'"
+
+- name: Delete temp directory
+  file:
+    name: "{{ mktemp.stdout }}"
+    state: absent
+  changed_when: False
+  check_mode: no