From 054c2a9f169c5547458a4e168855aeb4812b5797 Mon Sep 17 00:00:00 2001 From: ewolinetz Date: Fri, 13 Jan 2017 16:25:16 -0600 Subject: Updating upgrade_logging to be more idempotent --- roles/openshift_logging/README.md | 4 +++- roles/openshift_logging/defaults/main.yml | 2 ++ roles/openshift_logging/files/es_migration.sh | 2 -- roles/openshift_logging/tasks/install_curator.yaml | 10 ++++++---- roles/openshift_logging/tasks/install_kibana.yaml | 6 ++++-- roles/openshift_logging/tasks/start_cluster.yaml | 4 ++-- roles/openshift_logging/tasks/upgrade_logging.yaml | 22 +++++++++++++++------- 7 files changed, 32 insertions(+), 18 deletions(-) (limited to 'roles') diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md index 9836fc217..2cc2c48ee 100644 --- a/roles/openshift_logging/README.md +++ b/roles/openshift_logging/README.md @@ -31,7 +31,7 @@ When both `openshift_logging_install_logging` and `openshift_logging_upgrade_log - `openshift_logging_curator_script_log_level`: The script log level for Curator. Defaults to 'INFO'. - `openshift_logging_curator_log_level`: The log level for the Curator process. Defaults to 'ERROR'. - `openshift_logging_curator_cpu_limit`: The amount of CPU to allocate to Curator. Default is '100m'. -- `openshift_logging_curator_memory_limit`: The amount of memor to allocate to Curator. Unset if not specified. +- `openshift_logging_curator_memory_limit`: The amount of memory to allocate to Curator. Unset if not specified. - `openshift_logging_kibana_hostname`: The Kibana hostname. Defaults to 'kibana.example.com'. - `openshift_logging_kibana_cpu_limit`: The amount of CPU to allocate to Kibana or unset if not specified. @@ -39,6 +39,7 @@ When both `openshift_logging_install_logging` and `openshift_logging_upgrade_log - `openshift_logging_kibana_proxy_debug`: When "True", set the Kibana Proxy log level to DEBUG. Defaults to 'false'. - `openshift_logging_kibana_proxy_cpu_limit`: The amount of CPU to allocate to Kibana proxy or unset if not specified. - `openshift_logging_kibana_proxy_memory_limit`: The amount of memory to allocate to Kibana proxy or unset if not specified. +- `openshift_logging_kibana_replica_count`: The number of replicas Kibana should be scaled up to. Defaults to 1. - `openshift_logging_fluentd_nodeselector`: The node selector that the Fluentd daemonset uses to determine where to deploy to. Defaults to '"logging-infra-fluentd": "true"'. - `openshift_logging_fluentd_cpu_limit`: The CPU limit for Fluentd pods. Defaults to '100m'. @@ -84,3 +85,4 @@ same as above for their non-ops counterparts, but apply to the OPS cluster insta - `openshift_logging_kibana_ops_memory_limit`: The amount of memory to allocate to Kibana or unset if not specified. - `openshift_logging_kibana_ops_proxy_cpu_limit`: The amount of CPU to allocate to Kibana proxy or unset if not specified. - `openshift_logging_kibana_ops_proxy_memory_limit`: The amount of memory to allocate to Kibana proxy or unset if not specified. +- `openshift_logging_kibana_ops_replica_count`: The number of replicas Kibana ops should be scaled up to. Defaults to 1. diff --git a/roles/openshift_logging/defaults/main.yml b/roles/openshift_logging/defaults/main.yml index 4eb852207..919c53787 100644 --- a/roles/openshift_logging/defaults/main.yml +++ b/roles/openshift_logging/defaults/main.yml @@ -25,6 +25,7 @@ openshift_logging_kibana_memory_limit: null openshift_logging_kibana_proxy_debug: false openshift_logging_kibana_proxy_cpu_limit: null openshift_logging_kibana_proxy_memory_limit: null +openshift_logging_kibana_replica_count: 1 openshift_logging_kibana_ops_hostname: "kibana-ops.{{openshift.common.dns_domain}}" openshift_logging_kibana_ops_cpu_limit: null @@ -32,6 +33,7 @@ openshift_logging_kibana_ops_memory_limit: null openshift_logging_kibana_ops_proxy_debug: false openshift_logging_kibana_ops_proxy_cpu_limit: null openshift_logging_kibana_ops_proxy_memory_limit: null +openshift_logging_kibana_ops_replica_count: 1 openshift_logging_fluentd_nodeselector: {'logging-infra-fluentd': 'true'} openshift_logging_fluentd_cpu_limit: 100m diff --git a/roles/openshift_logging/files/es_migration.sh b/roles/openshift_logging/files/es_migration.sh index cca283bae..339b5a1b2 100644 --- a/roles/openshift_logging/files/es_migration.sh +++ b/roles/openshift_logging/files/es_migration.sh @@ -1,5 +1,3 @@ -#! bin/bash - CA=${1:-/etc/openshift/logging/ca.crt} KEY=${2:-/etc/openshift/logging/system.admin.key} CERT=${3:-/etc/openshift/logging/system.admin.crt} diff --git a/roles/openshift_logging/tasks/install_curator.yaml b/roles/openshift_logging/tasks/install_curator.yaml index 35116ae2b..8f2825552 100644 --- a/roles/openshift_logging/tasks/install_curator.yaml +++ b/roles/openshift_logging/tasks/install_curator.yaml @@ -1,5 +1,6 @@ --- -- command: > +- name: Check Curator current replica count + command: > {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-curator -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}} register: curator_replica_count @@ -7,7 +8,8 @@ ignore_errors: yes changed_when: no -- command: > +- name: Check Curator ops current replica count + command: > {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-curator-ops -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}} register: curator_ops_replica_count @@ -28,7 +30,7 @@ es_port: "{{openshift_logging_es_port}}" curator_cpu_limit: "{{openshift_logging_curator_cpu_limit }}" curator_memory_limit: "{{openshift_logging_curator_memory_limit }}" - replicas: "{{curator_replica_count.stdout | default (1)}}" + replicas: "{{curator_replica_count.stdout | default (0)}}" check_mode: no changed_when: no @@ -43,7 +45,7 @@ es_port: "{{openshift_logging_es_ops_port}}" curator_cpu_limit: "{{openshift_logging_curator_ops_cpu_limit }}" curator_memory_limit: "{{openshift_logging_curator_ops_memory_limit }}" - replicas: "{{curator_ops_replica_count.stdout | default (1)}}" + replicas: "{{curator_ops_replica_count.stdout | default (0)}}" when: openshift_logging_use_ops check_mode: no changed_when: no diff --git a/roles/openshift_logging/tasks/install_kibana.yaml b/roles/openshift_logging/tasks/install_kibana.yaml index f4ce85f97..de4b018dd 100644 --- a/roles/openshift_logging/tasks/install_kibana.yaml +++ b/roles/openshift_logging/tasks/install_kibana.yaml @@ -1,5 +1,6 @@ --- -- command: > +- name: Check Kibana current replica count + command: > {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-kibana -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}} register: kibana_replica_count @@ -7,7 +8,8 @@ ignore_errors: yes changed_when: no -- command: > +- name: Check Kibana ops current replica count + command: > {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-kibana-ops -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}} register: kibana_ops_replica_count diff --git a/roles/openshift_logging/tasks/start_cluster.yaml b/roles/openshift_logging/tasks/start_cluster.yaml index 4ce6d1aa7..a96ad3f3a 100644 --- a/roles/openshift_logging/tasks/start_cluster.yaml +++ b/roles/openshift_logging/tasks/start_cluster.yaml @@ -39,7 +39,7 @@ - name: start kibana include: scale.yaml vars: - desired: 1 + desired: "{{ openshift_logging_kibana_replica_count | default (1) }}" with_items: "{{kibana_dc.stdout_lines}}" loop_control: loop_var: object @@ -82,7 +82,7 @@ - name: start kibana-ops include: scale.yaml vars: - desired: 1 + desired: "{{ openshift_logging_kibana_ops_replica_count | default (1) }}" with_items: "{{kibana_dc.stdout_lines}}" loop_control: loop_var: object diff --git a/roles/openshift_logging/tasks/upgrade_logging.yaml b/roles/openshift_logging/tasks/upgrade_logging.yaml index 9b285a5fe..a93463239 100644 --- a/roles/openshift_logging/tasks/upgrade_logging.yaml +++ b/roles/openshift_logging/tasks/upgrade_logging.yaml @@ -7,7 +7,7 @@ vars: start_cluster: False -# ensure that ES is running +# start ES so that we can run migrate script - command: > {{openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig get dc -l component=es -o name -n {{openshift_logging_namespace}} register: es_dc @@ -21,13 +21,21 @@ loop_control: loop_var: object -- copy: - src: es_migration.sh - dest: {{mktemp.stdout}}/es_migration.sh +- command: > + {{ openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig get pods -n {{openshift_logging_namespace}} -l component=es -o jsonpath='{.items[?(@.status.phase == "Running")].metadata.name}' + register: running_pod + until: running_pod.stdout != '' + retries: 30 + delay: 10 + changed_when: no + check_mode: no -- name: Run upgrade scripts - command: > - sh {{mktemp.stdout}}/es_migration.sh {{openshift.common.config_base}}/logging/ca.crt {{openshift.common.config_base}}/logging/system.admin.key {{openshift.common.config_base}}/logging/system.admin.crt {{openshift_logging_es_host}} {{openshift_logging_es_port}} {{openshift_logging_namespace}} +- name: Run upgrade script + script: es_migration.sh {{openshift.common.config_base}}/logging/ca.crt {{openshift.common.config_base}}/logging/system.admin.key {{openshift.common.config_base}}/logging/system.admin.crt {{openshift_logging_es_host}} {{openshift_logging_es_port}} {{openshift_logging_namespace}} + register: script_output + changed_when: + - script_output.rc == 0 + - script_output.stdout.find("skipping update_for_uuid") == -1 or script_output.stdout.find("skipping update_for_common_data_model") == -1 - name: Start up rest of cluster include: start_cluster.yaml -- cgit v1.2.1