From e4751f88e52aa8e89e4c94bc6fe4c3346eccf6fe Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Tue, 20 Feb 2018 15:10:45 +0100 Subject: Handling GlusterFS storage security in OpenShift containers --- playbooks/ands-setup-vm.yml | 5 --- playbooks/openshift-setup-projects.yml | 1 + playbooks/openshift-setup-security.yml | 24 ++++++++++ roles/ands_kaas/tasks/file.yml | 18 +++++++- roles/ands_kaas/templates/0-gfs-volumes.yml.j2 | 9 ++-- roles/ands_kaas/templates/6-kaas-pods.yml.j2 | 35 ++++++++++++--- roles/ands_openshift/defaults/main.yml | 3 +- roles/ands_openshift/tasks/security.yml | 3 ++ roles/ands_openshift/tasks/security_resources.yml | 54 +++++++++++++++++++++++ roles/ands_openshift/tasks/users_resources.yml | 14 ++++-- roles/glusterfs/tasks/data | 1 + roles/glusterfs/tasks/db/vols3.yml | 14 ++++++ roles/openshift_resource/defaults/main.yml | 3 +- roles/openshift_resource/tasks/patch.yml | 41 +++++++++++++++++ setup.sh | 3 ++ setup/configs/openshift.yml | 3 +- setup/configs/security.yml | 21 +++++++++ setup/configs/volumes.yml | 13 ++++-- setup/projects/adei/vars/pods.yml | 52 ++++++++++++++++++++++ 19 files changed, 289 insertions(+), 28 deletions(-) delete mode 100644 playbooks/ands-setup-vm.yml create mode 100644 playbooks/openshift-setup-security.yml create mode 100644 roles/ands_openshift/tasks/security.yml create mode 100644 roles/ands_openshift/tasks/security_resources.yml create mode 120000 roles/glusterfs/tasks/data create mode 100644 roles/glusterfs/tasks/db/vols3.yml create mode 100644 roles/openshift_resource/tasks/patch.yml create mode 100644 setup/configs/security.yml create mode 100644 setup/projects/adei/vars/pods.yml diff --git a/playbooks/ands-setup-vm.yml b/playbooks/ands-setup-vm.yml deleted file mode 100644 index d97916d..0000000 --- a/playbooks/ands-setup-vm.yml +++ /dev/null @@ -1,5 +0,0 @@ -- name: Common setup procedures - hosts: vagrant - roles: - - role: ands_vagrant_vm - diff --git a/playbooks/openshift-setup-projects.yml b/playbooks/openshift-setup-projects.yml index a8af9c1..cc36498 100644 --- a/playbooks/openshift-setup-projects.yml +++ b/playbooks/openshift-setup-projects.yml @@ -15,6 +15,7 @@ hosts: masters roles: - { role: ands_openshift, subrole: users } + - { role: ands_openshift, subrole: security } - { role: ands_openshift, subrole: storage } - { role: ands_kaas } vars: diff --git a/playbooks/openshift-setup-security.yml b/playbooks/openshift-setup-security.yml new file mode 100644 index 0000000..6c85602 --- /dev/null +++ b/playbooks/openshift-setup-security.yml @@ -0,0 +1,24 @@ +- name: Configure users + hosts: masters + roles: + - { role: ands_facts } + + +- name: Temporary provision /etc/hosts with Masters IP. + hosts: nodes:!masters + tasks: + - lineinfile: dest="/etc/hosts" line="{{ ands_openshift_network | ipaddr(node_id) | ipaddr('address') }} {{ ands_openshift_lb }}" regexp=".*{{ ands_openshift_lb }}$" state="present" + when: (ands_provision_without_dns | default(false)) + vars: + node_id: "{{ hostvars[groups['masters'][0]]['ands_host_id'] }}" + +- name: Configure security + hosts: masters + roles: + - { role: ands_openshift, subrole: security } + +- name: Remove temporary entries in /etc/hosts + hosts: nodes:!masters + tasks: + - lineinfile: dest="/etc/hosts" regexp=".*{{ ands_openshift_lb }}$" state="absent" + when: (ands_provision_without_dns | default(false)) diff --git a/roles/ands_kaas/tasks/file.yml b/roles/ands_kaas/tasks/file.yml index 9a36e74..479ec68 100644 --- a/roles/ands_kaas/tasks/file.yml +++ b/roles/ands_kaas/tasks/file.yml @@ -1,9 +1,23 @@ --- +- name: Set group + set_fact: group="{{ file.group | default(kaas_project_config.file_group | default(ands_default_file_group)) }}" + +- name : Resolve project groups + set_fact: group="{{ (kaas_project_config.gids | default(ands_openshift_gids))[group].id }}" + when: group in ( kaas_project_config.gids | default(ands_openshift_gids) ) + +- name: Set owner + set_fact: owner="{{ file.owner | default(kaas_project_config.file_owner | default(ands_default_file_owner)) }}" + +- name : Resolve project uids + set_fact: owner="{{ (kaas_project_config.uids | default(ands_openshift_uids) )[owner].id }}" + when: owner in ( kaas_project_config.uids | default(ands_openshift_uids) ) + - name: "Setting up files in {{ path }}" file: path: "{{ path }}" recurse: "{{ file.recurse | default(true) }}" mode: "{{ file.mode | default( ((file.state | default('directory')) == 'directory') | ternary('0755', '0644') ) }}" - owner: "{{ file.owner | default(kaas_project_config.file_owner) | default(kaas_default_file_owner) }}" - group: "{{ file.group | default(kaas_project_config.file_group) | default(kaas_default_file_group) }}" + owner: "{{ owner }}" + group: "{{ group }}" state: "{{ file.state | default('directory') }}" diff --git a/roles/ands_kaas/templates/0-gfs-volumes.yml.j2 b/roles/ands_kaas/templates/0-gfs-volumes.yml.j2 index a162c8b..8e5842a 100644 --- a/roles/ands_kaas/templates/0-gfs-volumes.yml.j2 +++ b/roles/ands_kaas/templates/0-gfs-volumes.yml.j2 @@ -7,10 +7,11 @@ metadata: descriptions: "KATRIN Volumes" objects: {% for name, vol in (kaas_project_config.volumes | default(kaas_openshift_volumes)).iteritems() %} +{% set oc_name = vol.name | default(name) | regex_replace('_','-') %} - apiVersion: v1 kind: PersistentVolume metadata: - name: {{ vol.name | default(name) }} + name: {{ oc_name }} spec: persistentVolumeReclaimPolicy: Retain glusterfs: @@ -22,14 +23,14 @@ objects: capacity: storage: {{ vol.capacity | default(kaas_default_volume_capacity) }} claimRef: - name: {{ vol.name | default(name) }} + name: {{ oc_name }} namespace: {{ kaas_project }} - apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ vol.name | default(name) }} + name: {{ oc_name }} spec: - volumeName: {{ vol.name | default(name) }} + volumeName: {{ oc_name }} accessModes: - {{ vol.access | default('ReadWriteMany') }} resources: diff --git a/roles/ands_kaas/templates/6-kaas-pods.yml.j2 b/roles/ands_kaas/templates/6-kaas-pods.yml.j2 index 479b343..d5418d3 100644 --- a/roles/ands_kaas/templates/6-kaas-pods.yml.j2 +++ b/roles/ands_kaas/templates/6-kaas-pods.yml.j2 @@ -36,7 +36,7 @@ objects: - apiVersion: v1 kind: Route metadata: - name: kaas + name: {{ pod.name | default(name) }} spec: host: {{ pod.service.host }} to: @@ -66,7 +66,7 @@ objects: - apiVersion: v1 kind: DeploymentConfig metadata: - name: kaas + name: {{ pod.name | default(name) }} spec: replicas: {{ pod.sched.replicas | default(1) }} selector: @@ -93,11 +93,32 @@ objects: {% for img in pod.images %} {% set imgidx = loop.index %} {% for vol in img.mappings %} + {% set oc_name = vol.name | default(name) | regex_replace('_','-') %} - name: vol-{{imgidx}}-{{loop.index}} persistentVolumeClaim: - claimName: {{ vol.name }} + claimName: {{ oc_name }} {% endfor %} {% endfor %} + {% endif %} + {% if (pod.groups is defined) or (pod.run_as is defined) %} + securityContext: + {% if (pod.run_as is defined) %} + {% if (kaas_project_config.uids | default(kaas_openshift_uids))[pod.run_as] is defined %} + - {{ (kaas_project_config.uids | default(kaas_openshift_uids))[pod.run_as].id }} + {% else %} + - pod.run_as + {% endif %} + {% endif %} + {% if (pod.groups is defined) %} + supplementalGroups: + {% for group in pod.groups %} + {% if (kaas_project_config.gids | default(kaas_openshift_gids))[group] is defined %} + - {{ (kaas_project_config.gids | default(kaas_openshift_gids))[group].id }} + {% else %} + - group + {% endif %} + {% endfor %} + {% endif %} {% endif %} containers: {% for img in pod.images %} @@ -118,10 +139,12 @@ objects: {% endif %} {% if img.env is defined %} env: - {% for env_name, env_val in img.env.iteritems() %} + {% for env_item in img.env %} + {% set env_name = env_item.name %} + {% set env_val = env_item.value %} {% set env_parts = (env_val | string).split('@') %} + - name: "{{ env_name }}" {% if env_parts[0] == "secret" %} - - name: {{ env_name }} {% set env_sec = (env_parts[1] | string).split('/') %} valueFrom: secretKeyRef: @@ -134,7 +157,7 @@ objects: name: {{ env_cm[0] }} key: {{ env_cm[1] }} {% else %} - value: {{ env_val }} + value: "{{ env_val }}" {% endif %} {% endfor %} {% endif %} diff --git a/roles/ands_openshift/defaults/main.yml b/roles/ands_openshift/defaults/main.yml index e473b98..b97b584 100644 --- a/roles/ands_openshift/defaults/main.yml +++ b/roles/ands_openshift/defaults/main.yml @@ -1,4 +1,4 @@ -openshift_common_subroles: "{{ [ 'hostnames', 'users', 'storage' ] }}" +openshift_common_subroles: "{{ [ 'hostnames', 'users', 'security', 'storage' ] }}" openshift_heketi_subroles: "{{ [ 'ssh', 'heketi' ] }}" openshift_all_subroles: "{{ ands_configure_heketi | default(False) | ternary(openshift_common_subroles + openshift_heketi_subroles, openshift_common_subroles) }}" @@ -9,5 +9,6 @@ openshift_namespace: "default" ssh_template_path: "{{ ands_paths.provision }}/ssh/" storage_template_path: "{{ ands_paths.provision }}/gfs/" heketi_template_path: "{{ ands_paths.provision }}/heketi/" +ands_openshift_patch_path: "{{ ands_paths.provision }}/patch/" openshift_storage_nodes: "{{ groups.storage_nodes | map('extract', hostvars, 'ands_storage_hostname') | list }}" diff --git a/roles/ands_openshift/tasks/security.yml b/roles/ands_openshift/tasks/security.yml new file mode 100644 index 0000000..b1f017b --- /dev/null +++ b/roles/ands_openshift/tasks/security.yml @@ -0,0 +1,3 @@ +- include_tasks: security_resources.yml + run_once: true + delegate_to: "{{ groups.masters[0] }}" diff --git a/roles/ands_openshift/tasks/security_resources.yml b/roles/ands_openshift/tasks/security_resources.yml new file mode 100644 index 0000000..5644723 --- /dev/null +++ b/roles/ands_openshift/tasks/security_resources.yml @@ -0,0 +1,54 @@ +--- +- name: Ensure OpenShift patch directory exists + file: path="{{ ands_openshift_patch_path }}" state="directory" mode=0644 owner=root group=root + +# No spaces in patch, otherwise escaping mess... +- name: Patch group range in project configuration + include_role: name="openshift_resource" tasks_from="patch.yml" + vars: + project: "{{ prj_item }}" + resource: "ns/{{ prj_item }}" + patch: '{"metadata":{"annotations":{"openshift.io/sa.scc.supplemental-groups":"{{ands_openshift_gid_ranges[prj_item]}}"}}}' + patch_path: "{{ ands_openshift_patch_path }}" + with_items: "{{ (ands_openshift_gid_ranges | default({})).keys() }}" + loop_control: + loop_var: prj_item + +- name: Patch uid range in project configuration + include_role: name="openshift_resource" tasks_from="patch.yml" + vars: + project: "{{ prj_item }}" + resource: "ns/{{ prj_item }}" + patch: '{"metadata":{"annotations":{"openshift.io/sa.scc.uid-range":"{{ands_openshift_uid_ranges[prj_item]}}"}}}' + patch_path: "{{ ands_openshift_patch_path }}" + with_items: "{{ (ands_openshift_uid_ranges | default({})).keys() }}" + loop_control: + loop_var: prj_item + +- name: Restrict supplementalGroups + include_role: name="openshift_resource" tasks_from="patch.yml" + vars: + project: "{{ prj_item }}" + resource: "scc/restricted" + modes: "{{ ands_openshift_gid_mode | default({}) }}" + mode: "{{ (modes[prj_item] is defined) | ternary(modes[prj_item], modes['ands_default'] | default(false)) }}" + patch: '{"supplementalGroups":{"type":"{{mode}}"}}' + patch_path: "{{ ands_openshift_patch_path }}" + when: mode != false + with_items: "{{ (ands_openshift_projects | default({})).keys() }}" + loop_control: + loop_var: prj_item + +- name: Configure runAsUser + include_role: name="openshift_resource" tasks_from="patch.yml" + vars: + project: "{{ prj_item }}" + resource: "scc/restricted" + modes: "{{ ands_openshift_uid_mode | default({}) }}" + mode: "{{ (modes[prj_item] is defined) | ternary(modes[prj_item], modes['ands_default'] | default(false)) }}" + patch: '{"runAsUser":{"type":"{{mode}}"}}' + patch_path: "{{ ands_openshift_patch_path }}" + when: mode != false + with_items: "{{ (ands_openshift_projects | default({})).keys() }}" + loop_control: + loop_var: prj_item diff --git a/roles/ands_openshift/tasks/users_resources.yml b/roles/ands_openshift/tasks/users_resources.yml index 35323cb..5bc748c 100644 --- a/roles/ands_openshift/tasks/users_resources.yml +++ b/roles/ands_openshift/tasks/users_resources.yml @@ -2,7 +2,9 @@ - name: Configure cluster roles command: "oc adm policy add-cluster-role-to-user {{ item.key.split('/')[0] }} {{ item.value.replace(' ','').split(',') | join(' ') }}" with_dict: "{{ ands_openshift_roles }}" - when: "{{ item.key.split('/') | length == 1 }}" + when: key_len == "1" + vars: + key_len: "{{ item.key.split('/') | length }}" - name: Get project list command: "oc get projects -o json" @@ -20,7 +22,9 @@ - name: Configure per project roles command: "oc adm policy add-role-to-user -n {{ item.key.split('/')[0] }} {{ item.key.split('/')[1] }} {{ item.value.replace(' ','').split(',') | join(' ') }}" with_dict: "{{ ands_openshift_roles }}" - when: "{{ item.key.split('/') | length == 2 }}" + when: key_len == "2" + vars: + key_len: "{{ item.key.split('/') | length }}" - name: Get user list command: "oc get users -o json" @@ -31,10 +35,12 @@ set_fact: removed_users="{{ results.stdout | from_json | json_query('items[*].metadata.name') | difference(ands_openshift_users.keys()) }}" when: (results | succeeded) -- name: Create missing projects +- name: Remove user authentication command: "oc delete identity htpasswd_auth:{{ item }}" with_items: "{{ removed_users | default([]) }}" -- name: Create missing projects +- name: Remove users command: "oc delete user {{ item }}" with_items: "{{ removed_users | default([]) }}" + + diff --git a/roles/glusterfs/tasks/data b/roles/glusterfs/tasks/data new file mode 120000 index 0000000..31bb52e --- /dev/null +++ b/roles/glusterfs/tasks/data @@ -0,0 +1 @@ +cfg \ No newline at end of file diff --git a/roles/glusterfs/tasks/db/vols3.yml b/roles/glusterfs/tasks/db/vols3.yml new file mode 100644 index 0000000..b1beacb --- /dev/null +++ b/roles/glusterfs/tasks/db/vols3.yml @@ -0,0 +1,14 @@ +--- +- name: "Create {{ name }} volume" + gluster_volume: + state: present + name: "{{ name }}" + cluster: "{{ domain_servers | join(',') }}" + disperses: "3" + redundancies: "1" + bricks: "{{ glusterfs_bricks_path }}/brick-{{ name }}" + transport: "{{ glusterfs_transport }}" + + +- name: "Start {{ name }} volume" + gluster_volume: state="started" name="{{ name }}" diff --git a/roles/openshift_resource/defaults/main.yml b/roles/openshift_resource/defaults/main.yml index ec44c4f..7994827 100644 --- a/roles/openshift_resource/defaults/main.yml +++ b/roles/openshift_resource/defaults/main.yml @@ -1 +1,2 @@ -template_path: "/mnt/provision/templates" +template_path: "{{ ands_paths.provision }}/templates" +patch_path: "{{ ands_paths.provision }}/patches" \ No newline at end of file diff --git a/roles/openshift_resource/tasks/patch.yml b/roles/openshift_resource/tasks/patch.yml new file mode 100644 index 0000000..e2bbcfa --- /dev/null +++ b/roles/openshift_resource/tasks/patch.yml @@ -0,0 +1,41 @@ +--- +- name: Lookup the specified resource + command: "oc get -n '{{project}}' '{{resource}}' -o json" + register: orig_result + changed_when: 0 + +- name: Lookup API version of the specified resource + command: "oc get -n '{{project}}' '{{resource}}' --template {{'{{' + '.apiVersion' + '}}'}}" + register: api_version + changed_when: 0 + +# Fucking ansible is making mess of escaping. Main problem it parses to objects strings starting with '{ ... }', but not with ' { ... }' +- name: Escaping patch + set_fact: xpatch='{{patch | to_json | regex_replace(" ","") | regex_replace("^", " ")}}' + +- name: Generate dummy patch {{resource}} in {{project}} + command: "oc patch -n '{{project}}' --patch ' {\"apiVersion\": \"{{api_version.stdout}}\"}' --local=true -f - -o json" + args: + stdin: " {{ orig_result.stdout_lines | join('') }}" + register: dummy_result + changed_when: 0 + +- name: Generate test patch {{resource}} in {{project}} + command: "oc patch -n '{{project}}' --patch '{{xpatch}}' --local=true -f - -o json" + args: + stdin: " {{ orig_result.stdout_lines | join('') }}" + register: patch_result + changed_when: 0 + +#- debug: msg="{{ dummy_result.stdout }}" +# when: dummy_result.stdout != patch_result.stdout + +#- debug: msg="{{ patch_result.stdout }}" +# when: dummy_result.stdout != patch_result.stdout + +- name: Patch {{resource}} in {{project}} + command: "oc patch -n '{{project}}' '{{resource}}' --patch '{{xpatch}}'" + register: result + changed_when: (result | succeeded) + when: dummy_result.stdout != patch_result.stdout + \ No newline at end of file diff --git a/setup.sh b/setup.sh index 2905cda..20174f1 100755 --- a/setup.sh +++ b/setup.sh @@ -47,6 +47,9 @@ case "${1}" in users) apply playbooks/openshift-setup-users.yml || exit 1 ;; + security) + apply playbooks/openshift-setup-security.yml || exit 1 + ;; storage) apply playbooks/openshift-setup-storage.yml || exit 1 ;; diff --git a/setup/configs/openshift.yml b/setup/configs/openshift.yml index 6b9995c..e2a2d6d 100644 --- a/setup/configs/openshift.yml +++ b/setup/configs/openshift.yml @@ -2,7 +2,8 @@ ands_openshift_projects: katrin: KArlsruhe TRItium Neutrino adei: ADEI - + +# test: Tesing ands_openshift_users: pdv: IPE Administation Account diff --git a/setup/configs/security.yml b/setup/configs/security.yml new file mode 100644 index 0000000..413f57e --- /dev/null +++ b/setup/configs/security.yml @@ -0,0 +1,21 @@ +ands_openshift_gid_mode: +# adei: "RunAsAny" + ands_default: "MustRunAs" + +#ands_openshift_uid_mode: +# ands_default: "MustRunAsRange" + +#ands_openshift_uid_ranges: + +ands_openshift_gid_ranges: + katrin: "5000/10" + adei: "5010/10" + +ands_openshift_uids: + kaas: { id: 6000 } + +ands_openshift_gids: + kaas: { id: 6000 } + +ands_default_file_group: root +ands_default_file_owner: root diff --git a/setup/configs/volumes.yml b/setup/configs/volumes.yml index d0ba063..d93f177 100644 --- a/setup/configs/volumes.yml +++ b/setup/configs/volumes.yml @@ -3,6 +3,8 @@ ands_paths: provision: /mnt/provision openshift: /mnt/openshift temporary: /mnt/temporary + databases: /mnt/databases + katrin_data: /mnt/katrin ands_heketi_domain: servers: "storage_nodes" @@ -14,11 +16,15 @@ ands_storage_domains: clients: "masters" volumes: provision: { type: "cfg", mount: "{{ ands_paths.provision }}" } - - servers: "storage_nodes" - clients: "nodes" - volumes: openshift: { type: "cfg", mount: "{{ ands_paths.openshift }}" } + databases: { type: "db", mount: "{{ ands_paths.databases }}" } temporary: { type: "tmp", mount: "{{ ands_paths.temporary }}" } + katrin_data: { type: "data", mount: "{{ ands_paths.katrin_data }}" } +# - servers: "storage_nodes" +# clients: "nodes" +# openshift: { type: "cfg", mount: "{{ ands_paths.openshift }}" } +# temporary: { type: "tmp", mount: "{{ ands_paths.temporary }}" } +# volumes: # - ovirt: # - pdv: @@ -31,7 +37,6 @@ ands_openshift_volumes: log: { volume: "temporary", path: "/log", write: true} tmp: { volume: "temporary", path: "/tmp", write: true} - # Global list, we only take things from the volume of project ands_openshift_files: - { osv: "log", path: "apache2-kaas", state: "directory", mode: "0777" } diff --git a/setup/projects/adei/vars/pods.yml b/setup/projects/adei/vars/pods.yml new file mode 100644 index 0000000..3b104ea --- /dev/null +++ b/setup/projects/adei/vars/pods.yml @@ -0,0 +1,52 @@ +volumes: + adei_etc: { volume: "openshift", path: "/adei/etc" } + adei_db: { volume: "databases", path: "/adei", write: true} + +gids: + adei: { id: 5010 } + adei_db: { id: 5011 } + +files: + - { osv: "adei_db", path: "mysql", state: "directory", group: "adei_db", mode: "0775" } + + +pods: + mysql: + service: { ports: [ 3306 ] } + sched: { replicas: 1, selector: { master: 1 } } + selector: { master: 1 } + groups: [ "adei_db" ] + images: + - image: "openshift/mysql-56-centos7" + env: + - { name: "MYSQL_USER", value: "adei" } + - { name: "MYSQL_PASSWORD", value: "adei" } + - { name: "MYSQL_DATABASE", value: "adei" } + mappings: + - { name: "adei_etc", path: "mysql", mount: "/etc/mysql" } + - { name: "adei_db", path: "mysql", mount: "/var/lib/mysql/data" } + probes: + - { port: 3306 } + phpmyadmin: + service: { host: "phpmyadmin.{{ openshift_master_default_subdomain }}", ports: [ 80/8080 ] } + sched: { replicas: 1 } + images: + - image: "chsa/phpmyadmin-centos:4" + env: + - { name: "DB_SERVICE_HOST", value: "mysql.adei.svc.cluster.local" } + - { name: "DB_SERVICE_PORT", value: "3306" } + - { name: "DB_SERVICE_CONTROL_USER", value: "pma" } + - { name: "DB_SERVICE_CONTROL_PASSWORD", value: "adei" } + - { name: "DB_EXTRA_HOSTS", value: "mysql.katrin.svc.cluster.local" } + probes: + - { port: 8080, path: '/' } + + + +#oc: +# - template: "[0-3]*" +# - template: "[4-6]*" +# - resource: "route/apache" +# oc: "expose svc/kaas --name apache --hostname=apache.{{ openshift_master_default_subdomain }}" +# - template: "*" + \ No newline at end of file -- cgit v1.2.1