summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSuren A. Chilingaryan <csa@suren.me>2018-02-25 04:28:36 +0100
committerSuren A. Chilingaryan <csa@suren.me>2018-02-25 04:28:36 +0100
commit782397f3de45fd1585ff0e5361ff5b61c24c5dd4 (patch)
tree006b168a02923353e56d5d26d6fe581805895ce1
parent0caad1cc2ddd16ef41a7fe870640207d69e21046 (diff)
downloadands-782397f3de45fd1585ff0e5361ff5b61c24c5dd4.tar.gz
ands-782397f3de45fd1585ff0e5361ff5b61c24c5dd4.tar.bz2
ands-782397f3de45fd1585ff0e5361ff5b61c24c5dd4.tar.xz
ands-782397f3de45fd1585ff0e5361ff5b61c24c5dd4.zip
ADEI cachers
-rw-r--r--setup/projects/adei/templates/60-adei.yml.j241
-rw-r--r--setup/projects/adei/vars/globals.yml143
2 files changed, 121 insertions, 63 deletions
diff --git a/setup/projects/adei/templates/60-adei.yml.j2 b/setup/projects/adei/templates/60-adei.yml.j2
index 88bfa89..41840e4 100644
--- a/setup/projects/adei/templates/60-adei.yml.j2
+++ b/setup/projects/adei/templates/60-adei.yml.j2
@@ -9,7 +9,7 @@ metadata:
openshift.io/documentation-url: "http://adei.info"
openshift.io/support-url: "http://adei.info"
labels:
- name: adei
+ app: "adei"
objects:
{% for name, cfg in adei_frontends.iteritems() %}
{% if (cfg.enabled | default(true)) %}
@@ -62,6 +62,8 @@ objects:
spec:
schedule: "{{ cfg.cron }}"
concurrencyPolicy: "Forbid"
+ successfulJobsHistoryLimit: "2"
+ failedJobsHistoryLimit: "2"
jobTemplate:
spec:
completions: "1"
@@ -98,7 +100,6 @@ objects:
{% endif %}
name: "{{ cfg.name }}"
labels:
- app: "adei"
name: "{{ cfg.name }}"
adei-type: "{{ pod_type }}"
adei-name: "{{ name }}"
@@ -171,31 +172,37 @@ parameters:
- name: setup
value: "autogen"
description: "ADEI setup"
+ - name: continuous_caching
+ value: "1"
+ description: "Stop cachers if no work or poll the source continously"
+ - name: enable_logs
+ value: "1"
+ description: "Enable detailed ADEI logging (large volume)"
+ - name: enable_debug
+ value: "0"
+ description: "Include debugging information in ADEI logs (huge volume)"
- name: adei_replicas
value: "2"
description: "Number of frontend replics"
- - name: cache_replicas
- value: "1"
- description: "Default number of backend caching replicas"
- name: apache_servers
value: "150"
- - name: adei_schedule_cache
+ - name: sched_parallel
+ value: "sources"
+ description: "Basic unit of scheduling (servers or sources)"
+ - name: cache_replicas
+ value: "2"
+ description: "Default number of backend caching replicas"
+ - name: cache_parallel
+ value: "sources"
+ description: "Type of caching parallelism by a cacher replica: (sources or groups)"
+ - name: update_schedule
value: "*/5 * * * *"
description: "Schedule of caching group updates"
- - name: adei_schedule_maintain
+ - name: maintain_schedule
value: "*/15 * * * *"
description: "Schedule of maintain tasks"
- - name: adei_schedule_clean
+ - name: clean_schedule
value: "9 3 * * *"
description: "Schedule of cleaning tasks"
- - name: cache_parallel
- value: "source"
- description: "Type of caching parallelism: group, source, server"
- - name: enable_logs
- value: "1"
- description: "Enable detailed ADEI logging (large volume)"
- - name: enable_debug
- value: "0"
- description: "Include debugging information in ADEI logs (huge volume)"
- name: "adei_revision"
value: "last:1"
diff --git a/setup/projects/adei/vars/globals.yml b/setup/projects/adei/vars/globals.yml
index 60cb212..91f5933 100644
--- a/setup/projects/adei/vars/globals.yml
+++ b/setup/projects/adei/vars/globals.yml
@@ -15,17 +15,18 @@ adei_pod_env:
value: "adei"
- name: "ADEI_PORTS"
value: "8080"
- - name: "ADEI_SETUP"
- value: "${setup}"
- name: "ADEI_ENABLED_SETUPS"
- value: "${setup}"
- - name: "ADEI_PARALLEL"
- value: "${cache_parallel}"
+ value: "${setup} ${setup}-logs"
- name: "ADEI_REVISION"
value: "${adei_revision}"
-
+ - name: "ADEI_PATH"
+ value: "/adei/src"
+ - name: "ADEI_URL"
+ value: "http://adei-${setup}.adei.svc.cluster.local/adei"
adei_prod_env:
+ - name: "ADEI_SETUP"
+ value: "${setup}"
- name: "ADEI_RELEASE"
value: "1"
- name: "ADEI_WRITE_LOGS"
@@ -36,6 +37,8 @@ adei_prod_env:
value: "${apache_servers}"
adei_log_env:
+ - name: "ADEI_SETUP"
+ value: "${setup}-logs"
- name: "ADEI_RELEASE"
value: "1"
- name: "ADEI_WRITE_LOGS"
@@ -46,6 +49,8 @@ adei_log_env:
value: "5"
adei_debug_env:
+ - name: "ADEI_SETUP"
+ value: "${setup}"
- name: "ADEI_RELEASE"
value: "0"
- name: "ADEI_WRITE_LOGS"
@@ -56,8 +61,38 @@ adei_debug_env:
value: "5"
adei_cron_env:
+ - name: "ADEI_SETUP"
+ value: "${setup}"
- name: "ADEI_SCHEDULER"
- value: "1"
+ value: "${sched_parallel}"
+
+adei_ccache_env:
+ - name: "ADEI_CONTINUOUS_CACHING"
+ value: "${continuous_caching}"
+
+adei_cache_env:
+ - name: "ADEI_SETUP"
+ value: "${setup}"
+ - name: "ADEI_PARALLEL"
+ value: "${cache_parallel}"
+ - name: "ADEI_IDLE_SLEEP"
+ value: "5"
+
+adei_arc_cache_env:
+ - name: "ADEI_SETUP"
+ value: "${setup}"
+ - name: "ADEI_PARALLEL"
+ value: "${cache_parallel}"
+ - name: "ADEI_IDLE_SLEEP"
+ value: "300"
+
+adei_log_cache_env:
+ - name: "ADEI_SETUP"
+ value: "${setup}-logs"
+ - name: "ADEI_PARALLEL"
+ value: "${sched_parallel}"
+ - name: "ADEI_IDLE_SLEEP"
+ value: "300"
adei_pod_vols:
@@ -79,9 +114,6 @@ adei_pod_vols:
- name: adei-log
persistentVolumeClaim:
claimName: adei-log
-# - name: adei-ovr
-# emptyDir: {}
-
adei_prod_mounts:
- name: adei-src
@@ -107,42 +139,7 @@ adei_pod_mounts:
subPath: "${setup}/apache2"
mountPath: /var/log/apache2
-# Only backends...
-# - name: adei-log
-# subPath: "${setup}/mail"
-# mountPath: /var/spool/mail
-
-#adei_prod_pod_mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
-#adei_dbg_pod_mounts: "{{ adei_dbg_mounts | union(adei_pod_mounts) }}"
-
adei_frontends:
- update:
- name: "adei-${setup}-update"
- cron: "${adei_schedule_cache}"
- cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei.cron.sh" ]
- env: "{{ adei_pod_env | union(adei_cron_env) }}"
- vols: "{{ adei_pod_vols }}"
- mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
- groups: [ "adei" ]
- enabled: true
- maintain:
- name: "adei-${setup}-maintain"
- cron: "${adei_schedule_cache}"
- cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei_manager.cron.sh" ]
- env: "{{ adei_pod_env | union(adei_cron_env) }}"
- vols: "{{ adei_pod_vols }}"
- mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
- groups: [ "adei" ]
- enabled: false
- clean:
- name: "adei-${setup}-clean"
- cron: "${adei_schedule_cache}"
- cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei_clean.cron.sh" ]
- env: "{{ adei_pod_env | union(adei_cron_env) }}"
- vols: "{{ adei_pod_vols }}"
- mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
- groups: [ "adei" ]
- enabled: false
frontend:
name: "adei-${setup}"
node: "adei-${setup}.{{ adei_domain }}"
@@ -165,9 +162,63 @@ adei_frontends:
logs:
name: "adei-${setup}-logs"
node: "adei-${setup}-logs.{{ adei_domain }}"
- replicas: 1
+ replicas: "${enabled_logs}"
env: "{{ adei_pod_env | union(adei_log_env) }}"
vols: "{{ adei_pod_vols }}"
mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
groups: [ "adei" ]
enabled: false
+ cacher:
+ name: "adei-${setup}-cacher"
+ replicas: "${cache_replicas}"
+ cmd: [ "/bin/bash", "/adei/src/scripts/system/cacher.sh" ]
+ env: "{{ adei_pod_env | union(adei_cache_env) | union(adei_ccache_env) }}"
+ vols: "{{ adei_pod_vols }}"
+ mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+ groups: [ "adei" ]
+ enabled: true
+ archive_cacher:
+ name: "adei-${setup}-archive-cacher"
+ replicas: "1"
+ cmd: [ "/bin/bash", "/adei/src/scripts/system/cacher.sh -m archive" ]
+ env: "{{ adei_pod_env | union(adei_arc_cache_env) }}"
+ vols: "{{ adei_pod_vols }}"
+ mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+ groups: [ "adei" ]
+ enabled: false
+ log_cacher:
+ name: "adei-${setup}-log-cacher"
+ replicas: "${enabled_logs}"
+ cmd: [ "/bin/bash", "/adei/src/scripts/system/cacher.sh" ]
+ env: "{{ adei_pod_env | union(adei_log_cache_env) }}"
+ vols: "{{ adei_pod_vols }}"
+ mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+ groups: [ "adei" ]
+ enabled: false
+ update:
+ name: "adei-${setup}-update"
+ cron: "${update_schedule}"
+ cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei.cron.sh" ]
+ env: "{{ adei_pod_env | union(adei_cron_env) | union(adei_ccache_env) }}"
+ vols: "{{ adei_pod_vols }}"
+ mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+ groups: [ "adei" ]
+ enabled: true
+ maintain:
+ name: "adei-${setup}-maintain"
+ cron: "${maintain_schedule}"
+ cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei_manager.cron.sh" ]
+ env: "{{ adei_pod_env | union(adei_cron_env) }}"
+ vols: "{{ adei_pod_vols }}"
+ mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+ groups: [ "adei" ]
+ enabled: false
+ clean:
+ name: "adei-${setup}-clean"
+ cron: "${clean_schedule}"
+ cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei_clean.cron.sh" ]
+ env: "{{ adei_pod_env | union(adei_cron_env) }}"
+ vols: "{{ adei_pod_vols }}"
+ mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+ groups: [ "adei" ]
+ enabled: false