From 076cd9cbfed1b3e596f827cebeec71e909749953 Mon Sep 17 00:00:00 2001 From: Brian Stajkowski Date: Mon, 11 Jul 2016 12:44:00 -0700 Subject: [PATCH] Docker Update Add dockerfiles for each piece of infrastructure. Support no password for mysql. Also added the local logs shared folder with support for local venv code but further testing is needed. Change-Id: I983299a389a7e3230625114c2629ed86d765daa0 --- .gitignore | 5 ++-- docker.neutron.conf | 2 +- docker_down.sh | 5 +++- docker_reload.sh | 6 +++- docker_up.sh | 20 ++++++------- infrastructure/docker/debian6/Dockerfile | 27 ++++++++++++++++++ infrastructure/docker/debian7/Dockerfile | 27 ++++++++++++++++++ infrastructure/docker/debian8/Dockerfile | 27 ++++++++++++++++++ .../docker/quark/Dockerfile | 5 +++- infrastructure/docker/redis-master/Dockerfile | 20 +++++++++++++ .../docker/redis-sentinel/Dockerfile | 22 +++++++++++++++ infrastructure/docker/redis-slave/Dockerfile | 28 +++++++++++++++++++ 12 files changed, 177 insertions(+), 17 deletions(-) create mode 100644 infrastructure/docker/debian6/Dockerfile create mode 100644 infrastructure/docker/debian7/Dockerfile create mode 100644 infrastructure/docker/debian8/Dockerfile rename Dockerfile => infrastructure/docker/quark/Dockerfile (91%) create mode 100644 infrastructure/docker/redis-master/Dockerfile create mode 100644 infrastructure/docker/redis-sentinel/Dockerfile create mode 100644 infrastructure/docker/redis-slave/Dockerfile diff --git a/.gitignore b/.gitignore index ed7985a..13b98d0 100644 --- a/.gitignore +++ b/.gitignore @@ -50,5 +50,6 @@ ChangeLog # Docs docs/build -# Vagrant -.vagrant/ +# Docker +quark_container_venv +quark_container_logs diff --git a/docker.neutron.conf b/docker.neutron.conf index 86ffbab..e73577c 100644 --- a/docker.neutron.conf +++ b/docker.neutron.conf @@ -9,7 +9,7 @@ use_syslog: "True" syslog_log_facility: "LOG_LOCAL0" [DATABASE] -connection = mysql://root:password@docker-mysql/neutron?charset=utf8 +connection = mysql://root:@docker-mysql/neutron?charset=utf8 reconnect_interval = 5 auto_create_schema = False min_pool_size = 100 diff --git a/docker_down.sh b/docker_down.sh index 0a2d61d..1d677d7 100755 --- a/docker_down.sh +++ b/docker_down.sh @@ -47,4 +47,7 @@ docker stop quark docker rm quark echo "Neutron/Quark: Complete" echo "-----------------------" -echo " " \ No newline at end of file +echo " " + +rm -rf ./quark_container_logs +rm -rf ./quark_container_venv \ No newline at end of file diff --git a/docker_reload.sh b/docker_reload.sh index 8efe53f..54b8cef 100755 --- a/docker_reload.sh +++ b/docker_reload.sh @@ -7,8 +7,12 @@ echo "Neutron/Quark: Complete" echo "-----------------------" echo " " +rm -rf ./quark_container_logs/* +rm -rf ./quark_container_venv/* + echo "Neutron/Quark: Starting Container..." -docker run -d -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark +docker run -d -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv/lib/python2.7/site-packages/quark -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark +# docker run --entrypoint /bin/bash -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark echo "Neutron/Quark: Waiting for Neutron to Start..." # Need to wait for DB to standup sleep 5 diff --git a/docker_up.sh b/docker_up.sh index 52ebf58..aa78424 100755 --- a/docker_up.sh +++ b/docker_up.sh @@ -2,22 +2,20 @@ # To build a new container # docker build -t quark . +mkdir ./quark_container_logs +mkdir ./quark_container_venv + echo "Mysql: Starting Container..." -docker run -d --restart=always -p 3306:3306 --name mysql -e MYSQL_ROOT_PASSWORD=password -d mysql +docker run -d --restart=always -p 3306:3306 --name mysql -e MYSQL_ROOT_PASSWORD=password mysql echo "Mysql: Waiting for Mysql to Start..." # Need to wait for DB to standup sleep 30 +docker exec mysql mysql -uroot -ppassword -e "set password = password('')" echo "Mysql: Complete - PORT 3306" echo "------------------------------------------" echo " " -echo "PHPMyAdmin: Starting Container..." -docker run -d --restart=always --name phpmyadmin --link mysql:mysql -p 8081:80 nazarpc/phpmyadmin -echo "PHPMyAdmin: Complete - http://localhost:8081" -echo "------------------------------------------" -echo " " - echo "ELK: Starting Container..." docker run --restart=always --name kibana -d -p 514:514 -p 514:514/udp -p 8083:5601 -v /etc/localtime:/etc/localtime:ro pschiffe/rsyslog-elasticsearch-kibana echo "ELK: Complete - http://localhost:8083" @@ -31,8 +29,8 @@ echo "------------------------------------------" echo " " echo "Redis: Starting Master Container..." -docker run -d --restart=always -p 80:80 -v ~/data/redis0:/data --name=redis stajkowski/redis-master -echo "Redis: Complete - PORT 80" +docker run -d --restart=always -p 6379:6379 -v ~/data/redis0:/data --name=redis stajkowski/redis-master +echo "Redis: Complete - PORT 6379" echo "------------------------------------------" echo " " @@ -43,8 +41,8 @@ echo "------------------------------------------" echo " " echo "Neutron/Quark: Starting Container..." -docker run -d -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark -# docker run --entrypoint /bin/bash -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark +docker run -d -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv/lib/python2.7/site-packages/quark -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark +# docker run --entrypoint /bin/bash -v $(pwd):/opt/quark -v ~/neutron:/opt/configs -v $(pwd)/quark_container_logs:/var/log/neutron -v $(pwd)/quark_container_venv:/opt/venv -p 9696:9696 --link mysql:docker-mysql --link kibana:docker-kibana --link rabbitmq:docker-rabbitmq --link redis-sentinel:docker-redis-sentinel --name quark stajkowski/quark echo "Neutron/Quark: Waiting for Neutron to Start..." # Need to wait for DB to standup sleep 5 diff --git a/infrastructure/docker/debian6/Dockerfile b/infrastructure/docker/debian6/Dockerfile new file mode 100644 index 0000000..9430796 --- /dev/null +++ b/infrastructure/docker/debian6/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:oldstable + +RUN apt-get update + +RUN apt-get install -y python2.7 + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 10 + +RUN apt-get install -y python-pip python-dev libmysqlclient-dev rpm bzip2 wget git build-essential libssl-dev libffi-dev + +RUN pip install --upgrade cffi pyasn1 + +RUN pip install virtualenv PyYAML jinja2 paramiko + +RUN git clone https://github.com/ansible/ansible.git + +RUN cd ansible + +RUN cd ansible && git checkout tags/v1.9.1-1 + +RUN cd ansible && make install + +RUN mkdir /etc/ansible + +RUN cd ansible && cp examples/ansible.cfg /etc/ansible/. + +ENTRYPOINT /bin/bash \ No newline at end of file diff --git a/infrastructure/docker/debian7/Dockerfile b/infrastructure/docker/debian7/Dockerfile new file mode 100644 index 0000000..c111735 --- /dev/null +++ b/infrastructure/docker/debian7/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:7 + +RUN apt-get update + +RUN apt-get install -y python2.7 + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 10 + +RUN apt-get install -y python-pip python-dev libmysqlclient-dev rpm bzip2 wget git build-essential libssl-dev libffi-dev + +RUN pip install --upgrade cffi pyasn1 + +RUN pip install virtualenv PyYAML jinja2 paramiko + +RUN git clone https://github.com/ansible/ansible.git + +RUN cd ansible + +RUN cd ansible && git checkout tags/v1.9.1-1 + +RUN cd ansible && make install + +RUN mkdir /etc/ansible + +RUN cd ansible && cp examples/ansible.cfg /etc/ansible/. + +ENTRYPOINT /bin/bash \ No newline at end of file diff --git a/infrastructure/docker/debian8/Dockerfile b/infrastructure/docker/debian8/Dockerfile new file mode 100644 index 0000000..c2a5672 --- /dev/null +++ b/infrastructure/docker/debian8/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:8 + +RUN apt-get update + +RUN apt-get install -y python2.7 + +RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 10 + +RUN apt-get install -y python-pip python-dev libmysqlclient-dev rpm bzip2 wget git build-essential libssl-dev libffi-dev + +RUN pip install --upgrade cffi pyasn1 + +RUN pip install virtualenv PyYAML jinja2 paramiko + +RUN git clone https://github.com/ansible/ansible.git + +RUN cd ansible + +RUN cd ansible && git checkout tags/v1.9.1-1 + +RUN cd ansible && make install + +RUN mkdir /etc/ansible + +RUN cd ansible && cp examples/ansible.cfg /etc/ansible/. + +ENTRYPOINT /bin/bash \ No newline at end of file diff --git a/Dockerfile b/infrastructure/docker/quark/Dockerfile similarity index 91% rename from Dockerfile rename to infrastructure/docker/quark/Dockerfile index 7626631..d1e7622 100644 --- a/Dockerfile +++ b/infrastructure/docker/quark/Dockerfile @@ -7,7 +7,7 @@ RUN pip install virtualenv # Configure Rsyslog and Provision DB RUN echo 'local0.* @@docker-kibana:514' > /etc/rsyslog.d/60-neutron.conf RUN echo "service rsyslog restart" > ~/entrypoint.sh -RUN echo 'mysql -h docker-mysql -u root -ppassword -e "CREATE DATABASE neutron"' >> ~/entrypoint.sh +RUN echo 'mysql -h docker-mysql -u root -e "CREATE DATABASE neutron"' >> ~/entrypoint.sh # Update setuptools RUN wget https://bootstrap.pypa.io/ez_setup.py -O - | sudo python @@ -40,6 +40,9 @@ RUN echo "cd /opt/wafflehaus.neutron && git pull" >> ~/entrypoint.sh RUN echo "source /opt/venv/bin/activate && pip install -U -r /opt/wafflehaus.neutron/requirements.txt" >> ~/entrypoint.sh RUN echo "cd /opt/wafflehaus.neutron && source /opt/venv/bin/activate && python setup.py develop" >> ~/entrypoint.sh +# Install debug tools +RUN echo "source /opt/venv/bin/activate && pip install -U ipdb" >> ~/entrypoint.sh + # Put configuration files in place and start Neutron RUN echo "cp /opt/quark/docker.neutron.conf ~/neutron.conf" >> ~/entrypoint.sh RUN echo "cp /opt/quark/docker.apipaste.ini ~/api-paste.ini" >> ~/entrypoint.sh diff --git a/infrastructure/docker/redis-master/Dockerfile b/infrastructure/docker/redis-master/Dockerfile new file mode 100644 index 0000000..aee7dee --- /dev/null +++ b/infrastructure/docker/redis-master/Dockerfile @@ -0,0 +1,20 @@ +FROM stajkowski/redis-base:latest +ENV DEBIAN_FRONTEND noninteractive +##///*******************************************************/ +##// BELOW wil not work via +##// ENTRYPOINT["redis-server", "/etc/redis/redis.conf"] +##// ^^^ will reqiure intermediate bash.sh run to achieve this" +##// set redis default to bind to all nic's. +#//RUN sed -i 's/^\(bind .*\)$/# \1/' /etc/redis/redis.conf +#// #change ports as needed from 6379 to ... +#//RUN sed -i "s/port 6379/port 80/" /etc/redis/redis.conf +#///*******************************************************/ + +EXPOSE 6379 +VOLUME ["/data"] +ENTRYPOINT ["redis-server", "--port", "6379"] + +##////////////////////////////////////////////////////////////////// +# build: docker build -t vigour/redis-master . +# run: docker run -p 80:80 -v ~/data/redis0:/data -it --detach --name=redis0 vigour/redis-master +# cli: docker run -v -p 80:80 ~/data/redis0:/data -it --rm --name=redis0 vigour/redis-masters diff --git a/infrastructure/docker/redis-sentinel/Dockerfile b/infrastructure/docker/redis-sentinel/Dockerfile new file mode 100644 index 0000000..1effbe2 --- /dev/null +++ b/infrastructure/docker/redis-sentinel/Dockerfile @@ -0,0 +1,22 @@ +FROM stajkowski/redis-base:latest +ENV DEBIAN_FRONTEND noninteractive +##///*******************************************************/ +## set redis default to bind to all nic's. +RUN sed -i 's/^\(bind .*\)$/# \1/' /etc/redis/redis.conf +#change ports as needed from 6379 to ... +RUN echo "port 6380" > /etc/redis/sentinel.conf +RUN echo "sentinel monitor mymaster docker-redis 6379 2" >> /etc/redis/sentinel.conf +RUN echo "sentinel down-after-milliseconds mymaster 60000" >> /etc/redis/sentinel.conf +RUN echo "sentinel failover-timeout mymaster 180000" >> /etc/redis/sentinel.conf +RUN echo "sentinel parallel-syncs mymaster 1" >> /etc/redis/sentinel.conf + +##///*******************************************************/ +EXPOSE 6380 +VOLUME ["/data"] +WORKDIR /data +ENTRYPOINT ["redis-server", "/etc/redis/sentinel.conf", "--sentinel"] + +##////////////////////////////////////////////////////////////////// +# build: docker build -t vigour/redis-master . +# run: docker run -v ~/data/redis0:/data -it --detach --name=redis0 vigour/redis-master +# cli: docker run -v ~/data/redis0:/data -it --rm --name=redis0 vigour/redis-master diff --git a/infrastructure/docker/redis-slave/Dockerfile b/infrastructure/docker/redis-slave/Dockerfile new file mode 100644 index 0000000..915886d --- /dev/null +++ b/infrastructure/docker/redis-slave/Dockerfile @@ -0,0 +1,28 @@ +FROM vigour/redis-base:latest +ENV DEBIAN_FRONTEND noninteractive +##///*******************************************************/ +## set redis default to bind to all nic's. +RUN sed -i 's/^\(bind .*\)$/# \1/' /etc/redis/redis.conf +#change ports as needed from 6379 to ... +#RUN sed -i "s/port 6379/port 6380/" /etc/redis/redis.conf +##//------------------------ +## add a custom start script +##//------------------------ +RUN touch /start_slave.sh && chmod +x /start_slave.sh && \ +echo "#!/bin/bash" > /start_slave.sh > /start_slave.sh && \ +echo "#" >> /start_slave.sh && \ +echo "if [ -z \"$REDIS_MASTER_PORT_6379_TCP_ADDR\" ]; then" >> /start_slave.sh && \ +echo "echo \"REDIS_MASTER_PORT_6379_TCP_ADDR not defined. Did you run with -link?\";" >> /start_slave.sh && \ +echo "exit 7;" >> /start_slave.sh && \ +echo "fi" >> /start_slave.sh && \ +echo "# exec allows redis-server to receive signals for clean shutdown" >> /start_slave.sh && \ +echo "exec /usr/local/bin/redis-server --slaveof $REDIS_MASTER_PORT_6379_TCP_ADDR $REDIS_MASTER_PORT_6379_TCP_PORT $*" >> /start_slave.sh +##///*******************************************************/ +EXPOSE 6379 +VOLUME ["/data"] +ENTRYPOINT ["./start-slave.sh", "--dir", "/data"] + +##////////////////////////////////////////////////////////////////// +# build: docker build -t vigour/redis-slave . +# run: docker run -P --detach --name=redis0_slave --link=redis0:redis_master vigour/redis-slave +# cli: docker run -P -it --rm --name=redis0 vigour/redis-master