From 9c3d5c48694db1018d50aa85acad434f47e5ed0b Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Thu, 13 Dec 2018 00:05:49 +0100 Subject: Initial import --- .env | 1 + Makefile | 43 +++++++++++++++++ biomedisa/Dockerfile | 49 +++++++++++++++++++ biomedisa/Makefile | 48 +++++++++++++++++++ biomedisa/docker-entrypoint.sh | 16 +++++++ biomedisa/run-queues.sh | 9 ++++ biomedisa/supervisor/supervisord-cpu.conf | 27 +++++++++++ biomedisa/supervisor/supervisord-gpu.conf | 32 +++++++++++++ biomedisa/tests/test.sh | 4 ++ django/Dockerfile | 23 +++++++++ django/Makefile | 16 +++++++ django/docker-entrypoint.sh | 21 ++++++++ django/run-bootstrap.sh | 5 ++ django/run.sh | 3 ++ docker-compose.cpu.yml | 15 ++++++ docker-compose.yml | 66 ++++++++++++++++++++++++++ external/install.sh | 22 +++++++++ external/run.sh | 14 ++++++ external/tests/test.py | 12 +++++ external/tests/testmod.py | 11 +++++ mysql/Dockerfile | 4 ++ mysql/docker-entrypoint-initdb.d/biomedisa.sql | 1 + 22 files changed, 442 insertions(+) create mode 100644 .env create mode 100644 Makefile create mode 100644 biomedisa/Dockerfile create mode 100644 biomedisa/Makefile create mode 100755 biomedisa/docker-entrypoint.sh create mode 100755 biomedisa/run-queues.sh create mode 100644 biomedisa/supervisor/supervisord-cpu.conf create mode 100644 biomedisa/supervisor/supervisord-gpu.conf create mode 100755 biomedisa/tests/test.sh create mode 100644 django/Dockerfile create mode 100644 django/Makefile create mode 100755 django/docker-entrypoint.sh create mode 100755 django/run-bootstrap.sh create mode 100755 django/run.sh create mode 100644 docker-compose.cpu.yml create mode 100644 docker-compose.yml create mode 100644 external/install.sh create mode 100755 external/run.sh create mode 100644 external/tests/test.py create mode 100644 external/tests/testmod.py create mode 100644 mysql/Dockerfile create mode 100644 mysql/docker-entrypoint-initdb.d/biomedisa.sql diff --git a/.env b/.env new file mode 100644 index 0000000..06783cb --- /dev/null +++ b/.env @@ -0,0 +1 @@ +BIOMEDISA_HOST_SRC_PATH=/home/csa/biomedisa diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..60e89bb --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +# It seems start/stop does not respect '-p' option. We either need to avoid using it or skip -p + +.PHONY: run cpu start stop rebuild rebuild-cpu bootstrap volumes attach bash clean + +run: + docker-compose -p biomedisa up + +cpu: + docker-compose -p biomedisa -f docker-compose.yml -f docker-compose.cpu.yml up + +start: + docker-compose -p biomedisa up -d +# docker-compose -p biomedisa up --no-start +# docker-compose -p biomedisa start + +stop: + docker-compose -p biomedisa down +# docker-compose -p biomedisa stop + +rebuild: + docker-compose -p biomedisa build + +rebuild-cpu: + docker-compose -p biomedisa -f docker-compose.yml -f docker-compose.cpu.yml build + +bootstrap: + docker-compose -p biomedisa run --rm django /run-bootstrap.sh + +#volumes: +# for name in /var/lib/docker/volumes/biomedisa_*; do ln -s $name/_data/ `basename $name | cut -d '_' -f 2`; done + +attach: + docker attach `docker ps | grep "biomedisa_biomedisa " | cut -f 1 -d ' '` + +bash: + docker exec -it `docker ps | grep "biomedisa_biomedisa " | cut -f 1 -d ' '` bash + +clean: stop + c=`docker ps -a | grep "biomedisa_mysql " | cut -f 1 -d ' '`; \ + if [ -n "$$c" ]; then docker rm -f "$$c"; fi + v=`docker volume ls | grep "biomedisa_mysql"`; \ + if [ -n "$$v" ]; then docker volume rm biomedisa_mysql; fi + make bootstrap diff --git a/biomedisa/Dockerfile b/biomedisa/Dockerfile new file mode 100644 index 0000000..6666be7 --- /dev/null +++ b/biomedisa/Dockerfile @@ -0,0 +1,49 @@ +#FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 +FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 +MAINTAINER Suren A. Chilingaryan + +# Install Devel & Python +RUN apt-get update -qq \ + && apt-get install --no-install-recommends -y \ + git pkg-config netcat supervisor \ + telnet iputils-ping dnsutils \ + python python-dev python-pip python-setuptools python-virtualenv python-wheel python-numpy python-scipy python-h5py python-yaml python-pydot python-matplotlib libopenblas-base + +# Install Keras + Tensor Flow +ARG TENSORFLOW_VERSION=1.12.0 +ARG TENSORFLOW_DEVICE=gpu +ARG TENSORFLOW_APPEND=_gpu + +ARG KERAS_VERSION=2.2.4 +ENV KERAS_BACKEND=tensorflow + +RUN pip --no-cache-dir install \ + https://storage.googleapis.com/tensorflow/linux/${TENSORFLOW_DEVICE}/tensorflow${TENSORFLOW_APPEND}-${TENSORFLOW_VERSION}-cp27-none-linux_x86_64.whl + +RUN pip --no-cache-dir install --no-dependencies \ + git+https://github.com/fchollet/keras.git@${KERAS_VERSION} + +ADD tests/ /root/tests + +# install dependencies from debian packages +RUN apt-get install --no-install-recommends -y \ + mc \ + libsm6 libxrender-dev \ + openmpi-bin openmpi-doc libopenmpi-dev \ + python-gdcm python-mysqldb + +# upgrade pip +RUN pip --no-cache-dir install --upgrade pip + +# install dependencies from python packages +RUN pip --no-cache-dir install \ + setuptools==39.1.0 numpy==1.14.5 tifffile==0.14.0 SimpleParse==2.1.1 redis==2.10.6 rq==0.12 \ + scipy weave scikit-image opencv-python h5py pydicom Pillow django mpi4py pycuda + +COPY supervisor/*.conf /etc/ +COPY *-entrypoint.sh / +COPY run-*.sh / +ENTRYPOINT ["/docker-entrypoint.sh"] + +ENV BIOMEDISA_DEVICE=${TENSORFLOW_DEVICE} +CMD /usr/bin/supervisord -c /etc/supervisord-${BIOMEDISA_DEVICE}.conf diff --git a/biomedisa/Makefile b/biomedisa/Makefile new file mode 100644 index 0000000..c14b89e --- /dev/null +++ b/biomedisa/Makefile @@ -0,0 +1,48 @@ +all: build +#install: push + +.PHONY: rebuild build push start stop restart bash + +build: Dockerfile + docker build --tag chsa/biomedisa:latest . + +rebuild: Dockerfile + docker build --no-cache --tag chsa/biomedisa:latest . + +#push: build +# docker push chsa/biomedisa:latest + +start: build + @if [ `docker ps | grep adei-mysql | wc -l` -eq 0 ]; then \ + if [ `docker ps -a | grep adei-mysql | wc -l` -eq 0 ]; then \ + docker run --name adei-mysql -p 53306:3306/tcp -e MYSQL_ROOT_PASSWORD="adei" -e MYSQL_DATABASE="adei" -e MYSQL_USER="adei" -e MYSQL_PASSWORD="adei" -d mysql:5.7 ;\ + else \ + docker start adei-mysql ;\ + fi ;\ + fi ;\ + if [ `docker ps | grep chsa/adei | wc -l` -eq 0 ]; then \ + if [ `docker ps -a | grep adei-full | wc -l` -gt 0 ]; then \ + echo "Removing the stalled copy..." ;\ + docker rm adei-full ;\ + fi ;\ + docker run -p 8000:80 --name adei-full --link adei-mysql:adei-mysql -e ADEI_PORTS=8000 -t -d chsa/adei:latest ;\ + else \ + echo "Already running..." ;\ + fi + +stop: + @if [ `docker ps | grep adei-full | wc -l` -gt 0 ]; then \ + docker stop adei-full ;\ + docker rm adei-full ;\ + fi + +stop_mysql: stop + docker stop adei-mysql + +restart: + make stop + make start + +bash: build + docker run --runtime=nvidia -it chsa/biomedisa /bin/bash +# docker exec -it chsa/biomedisa /bin/bash diff --git a/biomedisa/docker-entrypoint.sh b/biomedisa/docker-entrypoint.sh new file mode 100755 index 0000000..a0f9cba --- /dev/null +++ b/biomedisa/docker-entrypoint.sh @@ -0,0 +1,16 @@ +#! /bin/bash + +set -e + +export PYTHONPATH=/biomedisa:${PYTHONPATH} +export DJANGO_SETTINGS_MODULE=biomedisa.settings + +if ! whoami &> /dev/null; then + if [ -w /etc/passwd ]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + fi +fi + +cd "/biomedisa" +echo "Running: $@" +exec "$@" diff --git a/biomedisa/run-queues.sh b/biomedisa/run-queues.sh new file mode 100755 index 0000000..a531b3b --- /dev/null +++ b/biomedisa/run-queues.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# start background workers +rq worker -u http://redis rq worker first_queue & +rq worker -u http://redis rq worker slices & +rq worker -u http://redis rq worker acwe & +rq worker -u http://redis rq worker cleanup & +rq worker -u http://redis rq worker share_notification & +rq worker -u http://redis rq worker stop_job & diff --git a/biomedisa/supervisor/supervisord-cpu.conf b/biomedisa/supervisor/supervisord-cpu.conf new file mode 100644 index 0000000..c942d7d --- /dev/null +++ b/biomedisa/supervisor/supervisord-cpu.conf @@ -0,0 +1,27 @@ +[supervisord] +nodaemon=true + +[program:slices] +command=rq worker -u http://redis rq worker slices +autorestart=true +exitcodes=0 + +[program:acwe] +command=rq worker -u http://redis rq worker acwe +autorestart=true +exitcodes=0 + +[program:cleanup] +command=rq worker -u http://redis rq worker cleanup +autorestart=true +exitcodes=0 + +[program:share_notification] +command=rq worker -u http://redis rq worker share_notification +autorestart=true +exitcodes=0 + +[program:stop_job] +command=rq worker -u http://redis rq worker stop_job +autorestart=true +exitcodes=0 diff --git a/biomedisa/supervisor/supervisord-gpu.conf b/biomedisa/supervisor/supervisord-gpu.conf new file mode 100644 index 0000000..94323d5 --- /dev/null +++ b/biomedisa/supervisor/supervisord-gpu.conf @@ -0,0 +1,32 @@ +[supervisord] +nodaemon=true + +[program:first] +command=rq worker -u http://redis rq worker first_queue +autorestart=true +exitcodes=0 + +[program:slices] +command=rq worker -u http://redis rq worker slices +autorestart=true +exitcodes=0 + +[program:acwe] +command=rq worker -u http://redis rq worker acwe +autorestart=true +exitcodes=0 + +[program:cleanup] +command=rq worker -u http://redis rq worker cleanup +autorestart=true +exitcodes=0 + +[program:share_notification] +command=rq worker -u http://redis rq worker share_notification +autorestart=true +exitcodes=0 + +[program:stop_job] +command=rq worker -u http://redis rq worker stop_job +autorestart=true +exitcodes=0 diff --git a/biomedisa/tests/test.sh b/biomedisa/tests/test.sh new file mode 100755 index 0000000..cf16ce1 --- /dev/null +++ b/biomedisa/tests/test.sh @@ -0,0 +1,4 @@ +#! /bin/bash + +python -c 'import tensorflow; print(tensorflow.__version__)' +python -c 'import keras; print(keras.__version__)' diff --git a/django/Dockerfile b/django/Dockerfile new file mode 100644 index 0000000..0f1e044 --- /dev/null +++ b/django/Dockerfile @@ -0,0 +1,23 @@ +FROM ubuntu:16.04 +MAINTAINER Suren A. Chilingaryan + +# Install Devel & Python +RUN apt-get update -qq \ + && apt-get install --no-install-recommends -y \ + gcc git pkg-config netcat \ + telnet iputils-ping dnsutils \ + python python-dev python-pip python-setuptools python-virtualenv python-wheel python-numpy python-scipy python-h5py python-yaml python-pydot python-matplotlib libopenblas-base \ + mysql-client libmysqlclient-dev + +# upgrade pip +RUN pip --no-cache-dir install --upgrade pip + +# install dependencies from python packages +RUN pip --no-cache-dir install \ + setuptools==39.1.0 numpy==1.14.5 tifffile==0.14.0 SimpleParse==2.1.1 redis==2.10.6 rq==0.12 \ + scipy weave scikit-image opencv-python h5py pydicom Pillow django MySQL + +COPY *-entrypoint.sh / +COPY run*.sh / +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["/run.sh"] diff --git a/django/Makefile b/django/Makefile new file mode 100644 index 0000000..78b94ac --- /dev/null +++ b/django/Makefile @@ -0,0 +1,16 @@ +all: build +#install: push + +.PHONY: rebuild build push start stop restart bash + +build: Dockerfile + docker build --tag chsa/biomedisa-django:latest . + +rebuild: Dockerfile + docker build --no-cache --tag chsa/biomedisa-django:latest . + +#push: build +# docker push chsa/biomedisa:latest + +bash: build + docker run -it chsa/biomedisa-django /bin/bash diff --git a/django/docker-entrypoint.sh b/django/docker-entrypoint.sh new file mode 100755 index 0000000..3a132a3 --- /dev/null +++ b/django/docker-entrypoint.sh @@ -0,0 +1,21 @@ +#! /bin/bash + +set -e + +export PYTHONPATH=/biomedisa:${PYTHONPATH} +export DJANGO_SETTINGS_MODULE=biomedisa.settings + +if ! whoami &> /dev/null; then + if [ -w /etc/passwd ]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + fi +fi + +until nc -z -v -w30 $MYSQL_HOST 3306; do + echo "Waiting for database connection..." + sleep 1 +done + +cd /biomedisa +echo "Running: $@" +exec "$@" diff --git a/django/run-bootstrap.sh b/django/run-bootstrap.sh new file mode 100755 index 0000000..c57325b --- /dev/null +++ b/django/run-bootstrap.sh @@ -0,0 +1,5 @@ +#! /bin/bash + +python manage.py makemigrations +python manage.py migrate +python manage.py createsuperuser diff --git a/django/run.sh b/django/run.sh new file mode 100755 index 0000000..f473012 --- /dev/null +++ b/django/run.sh @@ -0,0 +1,3 @@ +#! /bin/bash + +python manage.py runserver 0.0.0.0:8000 diff --git a/docker-compose.cpu.yml b/docker-compose.cpu.yml new file mode 100644 index 0000000..99d530f --- /dev/null +++ b/docker-compose.cpu.yml @@ -0,0 +1,15 @@ +version: '2.4' + +services: + redis: + ports: + - 127.0.0.1:6379:6379 + mysql: + ports: + - 127.0.0.1:3306:3306 + biomedisa: + build: + context: ./biomedisa + args: + TENSORFLOW_DEVICE: cpu + TENSORFLOW_APPEND: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1351b2a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,66 @@ +version: '2.4' + +services: + mysql: + build: ./mysql + restart: always + volumes: + - mysql:/var/lib/mysql + environment: + MYSQL_ROOT_PASSWORD: biomedisa + MYSQL_DATABASE: biomedisa_database + MYSQL_USER: biomedisa + MYSQL_PASSWORD: biomedisa + redis: + image: redis:2 + restart: always + django: + build: ./django + restart: always + tty: true + stdin_open: true + depends_on: + - mysql + - redis + links: + - mysql:mysql + - redis:redis + volumes: + - src:/biomedisa + - src:${BIOMEDISA_HOST_SRC_PATH} + environment: + MYSQL_HOST: mysql + REDIS_HOST: redis + BIOMEDISA_SRC_PATH: /biomedisa + ports: + - 8000:8000 + biomedisa: + build: ./biomedisa + runtime: nvidia + restart: always + tty: true + stdin_open: true + depends_on: + - mysql + - redis + links: + - mysql:mysql + - redis:redis + volumes: + - src:/biomedisa + - src:${BIOMEDISA_HOST_SRC_PATH} + environment: + MYSQL_HOST: mysql + REDIS_HOST: redis + BIOMEDISA_SRC_PATH: /biomedisa + NVIDIA_VISIBLE_DEVICES: all + +volumes: + src: + driver: local + driver_opts: + type: none + o: bind + device: ${BIOMEDISA_HOST_SRC_PATH} + mysql: +# media: diff --git a/external/install.sh b/external/install.sh new file mode 100644 index 0000000..3ad20f6 --- /dev/null +++ b/external/install.sh @@ -0,0 +1,22 @@ +#! /bin/bash + +yum install -y libtiff-devel +yum install -y MySQL-python python-devel python2-pip python2-setuptools python2-virtualenv numpy scipy h5py python2-pyyaml python2-pydot python2-matplotlib mysql-connector-python openblas +yum install -y openmpi openmpi-devel + +# upgrade pip +pip --no-cache-dir install --upgrade pip + +# extra pre-requisites +#pip --no-cache-dir install \ +# scipy decorator + +# conflicting packages +rpm -e scipy pyparsing python2-pydot python-matplotlib python2-matplotlib + +# install dependencies from python packages +export CC=/usr/lib64/openmpi/bin/mpicc + +pip --no-cache-dir install \ + setuptools==39.1.0 numpy==1.14.5 tifffile==0.14.0 SimpleParse==2.1.1 redis==2.10.6 rq==0.12 \ + weave scikit-image opencv-python h5py pydicom Pillow django mpi4py pycuda diff --git a/external/run.sh b/external/run.sh new file mode 100755 index 0000000..94490d7 --- /dev/null +++ b/external/run.sh @@ -0,0 +1,14 @@ +path=$(cat ../.env | grep -oP 'BIOMEDISA_HOST_SRC_PATH=\K.*') +( + cd "$path" || exit -1 + + export PYTHONPATH=$path:${PYTHONPATH} + export DJANGO_SETTINGS_MODULE=biomedisa.settings + + export BIOMEDISA_SRC_PATH=$path + export MYSQL_HOST="127.0.0.1" # localhost is treated as socket + export REDIS_HOST="localhost" + export PATH="$PATH:/usr/lib64/openmpi/bin/" + + LD_PRELOAD=/usr/lib64/openmpi/lib/libmpi.so rq worker -u http://localhost rq worker first_queue +) diff --git a/external/tests/test.py b/external/tests/test.py new file mode 100644 index 0000000..da9ff38 --- /dev/null +++ b/external/tests/test.py @@ -0,0 +1,12 @@ +from time import sleep +from redis import Redis +from rq import Queue + +from testmod import test + +q = Queue('slices', connection=Redis(host='localhost')) +j = q.enqueue(test) +print(j.result) +while j.result == None: + sleep(1) + print(j.result) diff --git a/external/tests/testmod.py b/external/tests/testmod.py new file mode 100644 index 0000000..e833d21 --- /dev/null +++ b/external/tests/testmod.py @@ -0,0 +1,11 @@ +import subprocess + +def test(): +# proc = subprocess.Popen(['env'], stdout=subprocess.PIPE) + proc = subprocess.Popen(['nvidia-smi'], stdout=subprocess.PIPE) +# proc = subprocess.Popen(['nvidia-smi', '-q', '-i', '0'], stdout=subprocess.PIPE) +# proc = subprocess.Popen(['ls', '/usr/bin/'], stdout=subprocess.PIPE) +# proc = subprocess.Popen('ls', stdout=subprocess.PIPE) + return proc.stdout.read() + + diff --git a/mysql/Dockerfile b/mysql/Dockerfile new file mode 100644 index 0000000..8a89e26 --- /dev/null +++ b/mysql/Dockerfile @@ -0,0 +1,4 @@ +FROM mysql:5.7 +MAINTAINER Suren A. Chilingaryan + +COPY docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/mysql/docker-entrypoint-initdb.d/biomedisa.sql b/mysql/docker-entrypoint-initdb.d/biomedisa.sql new file mode 100644 index 0000000..c437f0d --- /dev/null +++ b/mysql/docker-entrypoint-initdb.d/biomedisa.sql @@ -0,0 +1 @@ +GRANT ALL ON `biomedisa_database`.* TO 'biomedisa'@'%'; -- cgit v1.2.1