Adding support for ES 5.x tech preview opt in

9 files changed, 366 insertions, 4 deletions

diff --git a/roles/openshift_logging_fluentd/files/fluentd-throttle-config.yaml b/roles/openshift_logging_fluentd/files/2.x/fluentd-throttle-config.yaml
index 375621ff1..375621ff1 100644
--- a/roles/openshift_logging_fluentd/files/fluentd-throttle-config.yaml
+++ b/roles/openshift_logging_fluentd/files/2.x/fluentd-throttle-config.yaml
diff --git a/roles/openshift_logging_fluentd/files/secure-forward.conf b/roles/openshift_logging_fluentd/files/2.x/secure-forward.conf
index 87410c1c5..87410c1c5 100644
--- a/roles/openshift_logging_fluentd/files/secure-forward.conf
+++ b/roles/openshift_logging_fluentd/files/2.x/secure-forward.conf
diff --git a/roles/openshift_logging_fluentd/files/5.x/fluentd-throttle-config.yaml b/roles/openshift_logging_fluentd/files/5.x/fluentd-throttle-config.yaml
new file mode 100644
index 000000000..375621ff1
--- /dev/null
+++ b/roles/openshift_logging_fluentd/files/5.x/fluentd-throttle-config.yaml
@@ -0,0 +1,7 @@
+# Logging example fluentd throttling config file
+
+#example-project:
+#  read_lines_limit: 10
+#
+#.operations:
+#  read_lines_limit: 100
diff --git a/roles/openshift_logging_fluentd/files/5.x/secure-forward.conf b/roles/openshift_logging_fluentd/files/5.x/secure-forward.conf
new file mode 100644
index 000000000..87410c1c5
--- /dev/null
+++ b/roles/openshift_logging_fluentd/files/5.x/secure-forward.conf
@@ -0,0 +1,26 @@
+# <store>
+# @type secure_forward
+
+# self_hostname ${HOSTNAME}
+# shared_key <SECRET_STRING>
+
+# secure yes
+# enable_strict_verification yes
+
+# ca_cert_path /etc/fluent/keys/your_ca_cert
+# ca_private_key_path /etc/fluent/keys/your_private_key
+  # for private CA secret key
+# ca_private_key_passphrase passphrase
+
+# <server>
+  # or IP
+#   host server.fqdn.example.com
+#   port 24284
+# </server>
+# <server>
+  # ip address to connect
+#   host 203.0.113.8
+  # specify hostlabel for FQDN verification if ipaddress is used for host
+#   hostlabel server.fqdn.example.com
+# </server>
+# </store>
diff --git a/roles/openshift_logging_fluentd/tasks/main.yaml b/roles/openshift_logging_fluentd/tasks/main.yaml
index 79ebbca08..ef1c53de3 100644
--- a/roles/openshift_logging_fluentd/tasks/main.yaml
+++ b/roles/openshift_logging_fluentd/tasks/main.yaml
@@ -104,17 +104,17 @@
 
 # create Fluentd configmap
 - template:
-    src: fluent.conf.j2
+    src: "{{ __base_file_dir }}/fluent.conf.j2"
     dest: "{{ tempdir }}/fluent.conf"
   vars:
     deploy_type: "{{ openshift_logging_fluentd_deployment_type }}"
 
 - copy:
-    src: fluentd-throttle-config.yaml
+    src: "{{ __base_file_dir }}/fluentd-throttle-config.yaml"
     dest: "{{ tempdir }}/fluentd-throttle-config.yaml"
 
 - copy:
-    src: secure-forward.conf
+    src: "{{ __base_file_dir }}/secure-forward.conf"
     dest: "{{ tempdir }}/secure-forward.conf"
 
 - import_role:
@@ -161,7 +161,7 @@
 # TODO: pass in aggregation configurations
 - name: Generate logging-fluentd daemonset definition
   template:
-    src: fluentd.j2
+    src: "{{ __base_file_dir }}/fluentd.j2"
     dest: "{{ tempdir }}/templates/logging-fluentd.yaml"
   vars:
     daemonset_name: logging-fluentd
diff --git a/roles/openshift_logging_fluentd/templates/fluent.conf.j2 b/roles/openshift_logging_fluentd/templates/2.x/fluent.conf.j2
index 6e07b403a..6e07b403a 100644
--- a/roles/openshift_logging_fluentd/templates/fluent.conf.j2
+++ b/roles/openshift_logging_fluentd/templates/2.x/fluent.conf.j2
diff --git a/roles/openshift_logging_fluentd/templates/fluentd.j2 b/roles/openshift_logging_fluentd/templates/2.x/fluentd.j2
index c6256cf49..c6256cf49 100644
--- a/roles/openshift_logging_fluentd/templates/fluentd.j2
+++ b/roles/openshift_logging_fluentd/templates/2.x/fluentd.j2
diff --git a/roles/openshift_logging_fluentd/templates/5.x/fluent.conf.j2 b/roles/openshift_logging_fluentd/templates/5.x/fluent.conf.j2
new file mode 100644
index 000000000..6e07b403a
--- /dev/null
+++ b/roles/openshift_logging_fluentd/templates/5.x/fluent.conf.j2
@@ -0,0 +1,80 @@
+# This file is the fluentd configuration entrypoint. Edit with care.
+
+@include configs.d/openshift/system.conf
+
+# In each section below, pre- and post- includes don't include anything initially;
+# they exist to enable future additions to openshift conf as needed.
+
+## sources
+{% if deploy_type in ['hosted', 'secure-aggregator'] %}
+## ordered so that syslog always runs last...
+@include configs.d/openshift/input-pre-*.conf
+@include configs.d/dynamic/input-docker-*.conf
+@include configs.d/dynamic/input-syslog-*.conf
+@include configs.d/openshift/input-post-*.conf
+##
+{% else %}
+<source>
+  @type secure_forward
+  @label @INGRESS
+
+  self_hostname ${HOSTNAME}
+  bind 0.0.0.0
+  port {{openshift_logging_fluentd_aggregating_port}}
+
+  shared_key {{openshift_logging_fluentd_shared_key}}
+
+  secure {{openshift_logging_fluentd_aggregating_secure}}
+  enable_strict_verification {{openshift_logging_fluentd_aggregating_strict}}
+  ca_cert_path        {{openshift_logging_fluentd_aggregating_cert_path}}
+  ca_private_key_path {{openshift_logging_fluentd_aggregating_key_path}}
+  ca_private_key_passphrase {{openshift_logging_fluentd_aggregating_passphrase}}
+
+  <client>
+    host {{openshift_logging_fluentd_aggregating_host}}
+  </client>
+</source>
+{% endif %}
+
+<label @INGRESS>
+{% if deploy_type in ['hosted', 'secure-host'] %}
+## filters
+  @include configs.d/openshift/filter-pre-*.conf
+  @include configs.d/openshift/filter-retag-journal.conf
+  @include configs.d/openshift/filter-k8s-meta.conf
+  @include configs.d/openshift/filter-kibana-transform.conf
+  @include configs.d/openshift/filter-k8s-flatten-hash.conf
+  @include configs.d/openshift/filter-k8s-record-transform.conf
+  @include configs.d/openshift/filter-syslog-record-transform.conf
+  @include configs.d/openshift/filter-viaq-data-model.conf
+  @include configs.d/openshift/filter-post-*.conf
+##
+</label>
+
+<label @OUTPUT>
+## matches
+  @include configs.d/openshift/output-pre-*.conf
+  @include configs.d/openshift/output-operations.conf
+  @include configs.d/openshift/output-applications.conf
+  # no post - applications.conf matches everything left
+##
+{% else %}
+  <match **>
+    @type secure_forward
+
+    self_hostname ${HOSTNAME}
+    shared_key {{openshift_logging_fluentd_shared_key}}
+
+    secure {{openshift_logging_fluentd_aggregating_secure}}
+    enable_strict_verification {{openshift_logging_fluentd_aggregating_strict}}
+    ca_cert_path        {{openshift_logging_fluentd_aggregating_cert_path}}
+    ca_private_key_path {{openshift_logging_fluentd_aggregating_key_path}}
+    ca_private_key_passphrase {{openshift_logging_fluentd_aggregating_passphrase}}
+
+    <server>
+      host {{openshift_logging_fluentd_aggregating_host}}
+      port {{openshift_logging_fluentd_aggregating_port}}
+    </server>
+  </match>
+{% endif %}
+</label>
diff --git a/roles/openshift_logging_fluentd/templates/5.x/fluentd.j2 b/roles/openshift_logging_fluentd/templates/5.x/fluentd.j2
new file mode 100644
index 000000000..c6256cf49
--- /dev/null
+++ b/roles/openshift_logging_fluentd/templates/5.x/fluentd.j2
@@ -0,0 +1,249 @@
+apiVersion: extensions/v1beta1
+kind: "DaemonSet"
+metadata:
+  name: "{{ daemonset_name }}"
+  labels:
+    provider: openshift
+    component: "{{ daemonset_component }}"
+    logging-infra: "{{ daemonset_component }}"
+spec:
+  selector:
+    matchLabels:
+      provider: openshift
+      component: "{{ daemonset_component }}"
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      minReadySeconds: 600
+  template:
+    metadata:
+      name: "{{ daemonset_container_name }}"
+      labels:
+        logging-infra: "{{ daemonset_component }}"
+        provider: openshift
+        component: "{{ daemonset_component }}"
+    spec:
+      serviceAccountName: "{{ daemonset_serviceAccount }}"
+      nodeSelector:
+        {{ fluentd_nodeselector_key }}: "{{ fluentd_nodeselector_value }}"
+      containers:
+      - name: "{{ daemonset_container_name }}"
+        image: "{{ openshift_logging_fluentd_image_prefix }}{{ daemonset_name }}:{{ openshift_logging_fluentd_image_version }}"
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+{% if (fluentd_memory_limit is defined and fluentd_memory_limit is not none) or (fluentd_cpu_limit is defined and fluentd_cpu_limit is not none) or (fluentd_cpu_request is defined and fluentd_cpu_request is not none) %}
+        resources:
+{%   if (fluentd_memory_limit is defined and fluentd_memory_limit is not none) or (fluentd_cpu_limit is defined and fluentd_cpu_limit is not none) %}
+          limits:
+{%     if fluentd_cpu_limit is not none %}
+            cpu: "{{fluentd_cpu_limit}}"
+{%     endif %}
+{%     if fluentd_memory_limit is not none %}
+            memory: "{{fluentd_memory_limit}}"
+{%     endif %}
+{%   endif %}
+{%   if (fluentd_memory_limit is defined and fluentd_memory_limit is not none) or (fluentd_cpu_request is defined and fluentd_cpu_request is not none) %}
+          requests:
+{%     if fluentd_cpu_request is not none %}
+            cpu: "{{fluentd_cpu_request}}"
+{%     endif %}
+{%     if fluentd_memory_limit is not none %}
+            memory: "{{fluentd_memory_limit}}"
+{%     endif %}
+{%   endif %}
+{% endif %}
+        volumeMounts:
+        - name: runlogjournal
+          mountPath: /run/log/journal
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: config
+          mountPath: /etc/fluent/configs.d/user
+          readOnly: true
+        - name: certs
+          mountPath: /etc/fluent/keys
+          readOnly: true
+        - name: dockerhostname
+          mountPath: /etc/docker-hostname
+          readOnly: true
+        - name: localtime
+          mountPath: /etc/localtime
+          readOnly: true
+        - name: dockercfg
+          mountPath: /etc/sysconfig/docker
+          readOnly: true
+        - name: dockerdaemoncfg
+          mountPath: /etc/docker
+          readOnly: true
+        - name: filebufferstorage
+          mountPath: /var/lib/fluentd
+{% if openshift_logging_mux_client_mode is defined and
+     ((openshift_logging_mux_allow_external is defined and openshift_logging_mux_allow_external | bool) or
+      (openshift_logging_use_mux is defined and openshift_logging_use_mux | bool)) %}
+        - name: muxcerts
+          mountPath: /etc/fluent/muxkeys
+          readOnly: true
+{% endif %}
+        env:
+        - name: "K8S_HOST_URL"
+          value: "{{ openshift_logging_fluentd_master_url }}"
+        - name: "ES_HOST"
+          value: "{{ app_host }}"
+        - name: "ES_PORT"
+          value: "{{ app_port }}"
+        - name: "ES_CLIENT_CERT"
+          value: "{{ openshift_logging_fluentd_app_client_cert }}"
+        - name: "ES_CLIENT_KEY"
+          value: "{{ openshift_logging_fluentd_app_client_key }}"
+        - name: "ES_CA"
+          value: "{{ openshift_logging_fluentd_app_ca }}"
+        - name: "OPS_HOST"
+          value: "{{ ops_host }}"
+        - name: "OPS_PORT"
+          value: "{{ ops_port }}"
+        - name: "OPS_CLIENT_CERT"
+          value: "{{ openshift_logging_fluentd_ops_client_cert }}"
+        - name: "OPS_CLIENT_KEY"
+          value: "{{ openshift_logging_fluentd_ops_client_key }}"
+        - name: "OPS_CA"
+          value: "{{ openshift_logging_fluentd_ops_ca }}"
+        - name: "JOURNAL_SOURCE"
+          value: "{{ openshift_logging_fluentd_journal_source | default('') }}"
+        - name: "JOURNAL_READ_FROM_HEAD"
+          value: "{{ openshift_logging_fluentd_journal_read_from_head | lower }}"
+        - name: "BUFFER_QUEUE_LIMIT"
+          value: "{{ openshift_logging_fluentd_buffer_queue_limit }}"
+        - name: "BUFFER_SIZE_LIMIT"
+          value: "{{ openshift_logging_fluentd_buffer_size_limit }}"
+        - name: "FLUENTD_CPU_LIMIT"
+          valueFrom:
+            resourceFieldRef:
+              containerName: "{{ daemonset_container_name }}"
+              resource: limits.cpu
+        - name: "FLUENTD_MEMORY_LIMIT"
+          valueFrom:
+            resourceFieldRef:
+              containerName: "{{ daemonset_container_name }}"
+              resource: limits.memory
+        - name: "FILE_BUFFER_LIMIT"
+          value: "{{ openshift_logging_fluentd_file_buffer_limit | default('256Mi') }}"
+{% if openshift_logging_mux_client_mode is defined and
+     ((openshift_logging_mux_allow_external is defined and openshift_logging_mux_allow_external | bool) or
+      (openshift_logging_use_mux is defined and openshift_logging_use_mux | bool)) %}
+        - name: "MUX_CLIENT_MODE"
+          value: "{{ openshift_logging_mux_client_mode }}"
+{% endif %}
+{% if openshift_logging_install_eventrouter is defined and openshift_logging_install_eventrouter %}
+        - name: "TRANSFORM_EVENTS"
+          value: "true"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog is defined and openshift_logging_fluentd_remote_syslog %}
+        - name: USE_REMOTE_SYSLOG
+          value: "true"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog_host is defined %}
+        - name: REMOTE_SYSLOG_HOST
+          value: "{{ openshift_logging_fluentd_remote_syslog_host }}"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog_port is defined %}
+        - name: REMOTE_SYSLOG_PORT
+          value: "{{ openshift_logging_fluentd_remote_syslog_port }}"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog_severity is defined %}
+        - name: REMOTE_SYSLOG_SEVERITY
+          value: "{{ openshift_logging_fluentd_remote_syslog_severity }}"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog_facility is defined %}
+        - name: REMOTE_SYSLOG_FACILITY
+          value: "{{ openshift_logging_fluentd_remote_syslog_facility }}"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog_remove_tag_prefix is defined %}
+        - name: REMOTE_SYSLOG_REMOVE_TAG_PREFIX
+          value: "{{ openshift_logging_fluentd_remote_syslog_remove_tag_prefix }}"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog_tag_key is defined %}
+        - name: REMOTE_SYSLOG_TAG_KEY
+          value: "{{ openshift_logging_fluentd_remote_syslog_tag_key }}"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog_use_record is defined %}
+        - name: REMOTE_SYSLOG_USE_RECORD
+          value: "{{ openshift_logging_fluentd_remote_syslog_use_record }}"
+{% endif %}
+
+{% if openshift_logging_fluentd_remote_syslog_payload_key is defined %}
+        - name: REMOTE_SYSLOG_PAYLOAD_KEY
+          value: "{{ openshift_logging_fluentd_remote_syslog_payload_key }}"
+{% endif %}
+
+{% if audit_container_engine %}
+        - name: "AUDIT_CONTAINER_ENGINE"
+          value: "{{ audit_container_engine | lower }}"
+{% endif %}
+
+{% if audit_container_engine %}
+        - name: "NODE_NAME"
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+{% endif %}
+
+{% if audit_log_file != '' %}
+        - name: AUDIT_FILE
+          value: "{{ audit_log_file }}"
+{% endif %}
+
+{% if audit_pos_log_file != '' %}
+        - name: AUDIT_POS_FILE
+          value: "{{ audit_pos_log_file }}"
+{% endif %}
+
+      volumes:
+      - name: runlogjournal
+        hostPath:
+          path: /run/log/journal
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: config
+        configMap:
+          name: logging-fluentd
+      - name: certs
+        secret:
+          secretName: logging-fluentd
+      - name: dockerhostname
+        hostPath:
+          path: /etc/hostname
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      - name: dockercfg
+        hostPath:
+          path: /etc/sysconfig/docker
+      - name: dockerdaemoncfg
+        hostPath:
+          path: /etc/docker
+{% if openshift_logging_mux_client_mode is defined and
+     ((openshift_logging_mux_allow_external is defined and openshift_logging_mux_allow_external | bool) or
+      (openshift_logging_use_mux is defined and openshift_logging_use_mux | bool)) %}
+      - name: muxcerts
+        secret:
+          secretName: logging-mux
+{% endif %}
+      - name: filebufferstorage
+        hostPath:
+          path: "/var/lib/fluentd"