diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..c6a42acb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +*.egg* +*.py[cod] +.coverage +.testrepository/ +.tox/ +.venv/ +AUTHORS +ChangeLog +build/ +cover/ +dist +.git/ \ No newline at end of file diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..5afa7a6e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:14.04 + +RUN \ + groupadd dev && \ + useradd -g dev -s /bin/bash -d /home/dev -m dev && \ + ( umask 226 && echo "dev ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/50_dev ) + +ENV DEBIAN_FRONTEND noninteractive + +RUN \ + apt-get update && \ + apt-get -y install \ + sudo git vim wget \ + nginx \ + python-dev python3-dev python-pip \ + libmysqlclient-dev mysql-client-5.6 mysql-server-5.6 \ + npm && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/lib/mysql/* && \ + rm -rf /etc/nginx/sites-enabled/default + +RUN \ + pip install virtualenv tox ipython ipdb httpie && \ + npm install -g bower && \ + ln -s /usr/bin/nodejs /usr/bin/node + +ADD /docker/scripts/* /usr/bin/ +ADD . /refstack + +ENV PYTHONPATH=/home/dev/refstack \ + SQL_DIR=/home/dev/mysql + +ENV REFSTACK_MYSQL_URL="mysql+pymysql://root@localhost/refstack?unix_socket=${SQL_DIR}/mysql.socket&charset=utf8" + +USER dev + +RUN \ + echo "cd /home/dev/refstack" >> /home/dev/.bashrc &&\ + echo "alias activate='source /home/dev/refstack/.venv/bin/activate'" >> /home/dev/.bashrc &&\ + echo "alias mysql='mysql --no-defaults -S ${SQL_DIR}/mysql.socket'" >> /home/dev/.bashrc &&\ + start.sh &&\ + api-init-db + +CMD start.sh -s +EXPOSE 443 \ No newline at end of file diff --git a/docker/nginx/refstack-site.conf b/docker/nginx/refstack-site.conf new file mode 100644 index 00000000..eafa11ce --- /dev/null +++ b/docker/nginx/refstack-site.conf @@ -0,0 +1,19 @@ +server { + server_name 127.0.0.1; + listen 443 ssl; + + ssl on; + ssl_certificate /etc/nginx/certificates/refstack_dev.crt; + ssl_certificate_key /etc/nginx/certificates/refstack_dev.key; + ssl_protocols TLSv1.1 TLSv1.2; + ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5:!DSS:!RC4; + ssl_prefer_server_ciphers on; + ssl_session_timeout 5m; + location / { + access_log off; + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} \ No newline at end of file diff --git a/docker/nginx/refstack_dev.crt b/docker/nginx/refstack_dev.crt new file mode 100644 index 00000000..24b5ffee --- /dev/null +++ b/docker/nginx/refstack_dev.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICpzCCAhACCQCjqL+hsTaQ6zANBgkqhkiG9w0BAQsFADCBlzELMAkGA1UEBhMC +VUExEDAOBgNVBAgMB0toYXJraXYxEDAOBgNVBAcMB0toYXJraXYxHTAbBgNVBAoM +FE9wZW5zdGFjayBGb3VuZGF0aW9uMRswGQYDVQQDDBJyZWZzdGFjay5sb2NhbC5v +cmcxKDAmBgkqhkiG9w0BCQEWGXNzbHlwdXNoZW5rb0BtaXJhbnRpcy5jb20wHhcN +MTUwNTIxMTYyNDQ1WhcNMjUwNTE4MTYyNDQ1WjCBlzELMAkGA1UEBhMCVUExEDAO +BgNVBAgMB0toYXJraXYxEDAOBgNVBAcMB0toYXJraXYxHTAbBgNVBAoMFE9wZW5z +dGFjayBGb3VuZGF0aW9uMRswGQYDVQQDDBJyZWZzdGFjay5sb2NhbC5vcmcxKDAm +BgkqhkiG9w0BCQEWGXNzbHlwdXNoZW5rb0BtaXJhbnRpcy5jb20wgZ8wDQYJKoZI +hvcNAQEBBQADgY0AMIGJAoGBAMKfF4KpXa+/Ju5SM/oEuEKxffXh6WnA/eG4FQoP +1JMAKKn4wIsn4umKDHebKBDDIT/nlEuDQC9+dour1UxFhba8kJGh5QCmtp+qiWXj +H2f+U0RwBacHgjZoNOqJ+V88PW949IhD91v/lDYmDNtVUHt7BJw7nrnd5MLJAmBe +3S15AgMBAAEwDQYJKoZIhvcNAQELBQADgYEAU0WxG2amQsEv8qq4Vgps4zUTtnec +Vr6KMU06IrvNF0MEODJhasoQFmr2J6dy90abSqPPEdW76cxi1J6wtIEtNvW81elS +9OvdKwL+BoPFo+4G2VvT5Fj8DEl8goyIRGiK7+gpflS4jDRX57DVujgpVd5Omu7L +7F+OgXFZceBNBJw= +-----END CERTIFICATE----- diff --git a/docker/nginx/refstack_dev.key b/docker/nginx/refstack_dev.key new file mode 100644 index 00000000..146475ae --- /dev/null +++ b/docker/nginx/refstack_dev.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQDCnxeCqV2vvybuUjP6BLhCsX314elpwP3huBUKD9STACip+MCL +J+Lpigx3mygQwyE/55RLg0AvfnaLq9VMRYW2vJCRoeUAprafqoll4x9n/lNEcAWn +B4I2aDTqiflfPD1vePSIQ/db/5Q2JgzbVVB7ewScO5653eTCyQJgXt0teQIDAQAB +AoGAaPWDqGPOsslUJZMPlPaWqOEwHTsIto/uW5z7O8Ht0plzVLdin6mTJn/c2WRD +50ZU2DH8N/1A0FxTcl/pWIjl4wZPDOQ3W8EVcQ30gqV1vunXOi5jDGulCv0nsDXK +YifHxRDehr+ND20IqsQFv+k4PBBTcOMJ+7YpM+DrLubNAkECQQDmOsKF1jumAMP9 +CIkJ8wzXIzAk07w4QXLK1DMoSQVHI0Zz0KjCyJNNbR1w5J7c3QD4KWbIt/PSWuz/ +L+/G6YTjAkEA2Gf7AhFRv1cLg/l/1SwXtVb9MOh7Gf27XuTZeKyV202Fq0y6FhK/ +AQPPQfWQYcsrLkKwegIERtaY34ALLQPu8wJAUBsz4cOH35u2lc0peXfDCPwqXTX6 +8Iv9OAubfTHjDzx74AJDJfsKHc+Qhd5WVDzlgHNPWxl+UbvnaGcyg8BuxwJAXVAA +wPR83leHRKH5yA6aLnxS8prcMenhuFpPl6Q7ffOgdqu/9bKhn6tn3BYp6rEzbmAd +Po7OD0mLY5wPtZpjlwJAShmD4/1gjmV1aYAxQs6gPDDCr5oycn7jyta59gcwKdAv +zO5eKW1jMd+gg4jk3TiuLECdorUoGGbvIxEeP1gGBA== +-----END RSA PRIVATE KEY----- diff --git a/docker/scripts/api-db-version b/docker/scripts/api-db-version new file mode 100755 index 00000000..fbce84dc --- /dev/null +++ b/docker/scripts/api-db-version @@ -0,0 +1,2 @@ +#!/bin/bash +/home/dev/refstack/.venv/bin/python /home/dev/refstack/bin/refstack-manage version 2>/dev/null \ No newline at end of file diff --git a/docker/scripts/api-init-db b/docker/scripts/api-init-db new file mode 100755 index 00000000..edad5421 --- /dev/null +++ b/docker/scripts/api-init-db @@ -0,0 +1,5 @@ +#!/bin/bash +[[ ${DEBUG_MODE} ]] && set -x +mysql --no-defaults -S ${SQL_DIR}/mysql.socket -e 'CREATE DATABASE refstack;' +cd /home/dev/refstack +.venv/bin/python bin/refstack-manage upgrade --revision head \ No newline at end of file diff --git a/docker/scripts/api-sync b/docker/scripts/api-sync new file mode 100755 index 00000000..e4b27658 --- /dev/null +++ b/docker/scripts/api-sync @@ -0,0 +1,5 @@ +#!/bin/bash +[[ ${DEBUG_MODE} ]] && set -x +echo "Syncing project files..." +rsync -avr /refstack /home/dev --exclude-from '/refstack/.gitignore' > /dev/null && \ +echo "Done!" \ No newline at end of file diff --git a/docker/scripts/api-up b/docker/scripts/api-up new file mode 100755 index 00000000..861d557f --- /dev/null +++ b/docker/scripts/api-up @@ -0,0 +1,5 @@ +#!/bin/bash +[[ ${DEBUG_MODE} ]] && set -x +api-sync +cd /home/dev/refstack +.venv/bin/pecan serve refstack/api/config.py > /dev/null diff --git a/docker/scripts/start.sh b/docker/scripts/start.sh new file mode 100755 index 00000000..a98551c1 --- /dev/null +++ b/docker/scripts/start.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +wait_for_line () { + while read line; do + echo "$line" | grep -q "$1" && break + done < "$2" + # Read the fifo for ever otherwise process would block + cat "$2" >/dev/null & +} + +build_tmpl () { + TEMPLATE="$1" + TARGET="$2" + cat ${TEMPLATE} | \ + while read LINE; do + NEWLINE=$(eval echo ${LINE}) + [[ ! -z "$NEWLINE" ]] && echo ${NEWLINE} + done > ${TARGET} +} + +start_mysql () { + # Start MySQL process for tests + [ ! -d ${SQL_DIR} ] && mkdir ${SQL_DIR} + sudo chown dev:dev ${SQL_DIR} + rm -rf ${SQL_DIR}/out && mkfifo ${SQL_DIR}/out + rm -rf ${SQL_DIR}/mysql.socket + # On systems like Fedora here's where mysqld can be found + PATH=$PATH:/usr/libexec + mysqld --no-defaults --datadir=${SQL_DIR} --pid-file=${SQL_DIR}/mysql.pid \ + --socket=${SQL_DIR}/mysql.socket --skip-networking \ + --skip-grant-tables &> ${SQL_DIR}/out & + # Wait for MySQL to start listening to connections + wait_for_line "mysqld: ready for connections." ${SQL_DIR}/out +} + +build_refstack_env () { + api-sync + cd /home/dev/refstack + [ ! -d .venv ] && virtualenv .venv + .venv/bin/pip install -r requirements.txt + #Install some dev tools + .venv/bin/pip install ipython ipdb httpie + cd /home/dev/refstack + bower install --config.interactive=false + + build_tmpl /refstack/docker/templates/config.json.tmpl /home/dev/refstack/refstack-ui/app/config.json + build_tmpl /refstack/docker/templates/refstack.conf.tmpl /home/dev/refstack.conf + sudo cp /home/dev/refstack.conf /etc +} + +start_nginx () { + [ ! -d /etc/nginx/certificates ] && sudo mkdir /etc/nginx/certificates + sudo cp /refstack/docker/nginx/refstack_dev.key /etc/nginx/certificates + sudo cp /refstack/docker/nginx/refstack_dev.crt /etc/nginx/certificates + sudo cp /refstack/docker/nginx/refstack-site.conf /etc/nginx/sites-enabled/ + sudo nginx +} + +while getopts ":s" opt; do + case ${opt} in + s) SLEEP=true;; + esac +done + +[[ ${DEBUG_MODE} ]] && set -x + +touch /tmp/is-not-ready + +start_mysql +start_nginx +build_refstack_env + +rm -rf /tmp/is-not-ready + +if [[ ${SLEEP} ]]; then + set +x + sleep 1024d +fi diff --git a/docker/templates/config.json.tmpl b/docker/templates/config.json.tmpl new file mode 100644 index 00000000..c16d9ad3 --- /dev/null +++ b/docker/templates/config.json.tmpl @@ -0,0 +1 @@ +{\\"refstackApiUrl\\": \\"https://${REFSTACK_HOST:-127.0.0.1}/v1\\"} diff --git a/docker/templates/refstack.conf.tmpl b/docker/templates/refstack.conf.tmpl new file mode 100644 index 00000000..eb0eb9eb --- /dev/null +++ b/docker/templates/refstack.conf.tmpl @@ -0,0 +1,12 @@ +[DEFAULT] +debug = true +verbose = true + +[api] +static_root = /home/dev/refstack/refstack-ui/app +template_path = /home/dev/refstack/refstack-ui/app +app_dev_mode = true +test_results_url = https://${REFSTACK_HOST:-127.0.0.1}/#/results/%s + +[database] +connection = ${REFSTACK_MYSQL_URL} diff --git a/refstack/api/app.py b/refstack/api/app.py index 6dcaa720..b90977e1 100644 --- a/refstack/api/app.py +++ b/refstack/api/app.py @@ -65,7 +65,7 @@ API_OPTS = [ 'contain some details with debug information.' ), cfg.StrOpt('test_results_url', - default='http://refstack.net/output.html?test_id=%s', + default='http://refstack.net/#/results/%s', help='Template for test result url.' ), cfg.StrOpt('github_api_capabilities_url', @@ -187,4 +187,8 @@ def setup_app(config): )] ) + if CONF.api.app_dev_mode: + LOG.debug('\n\n Refstack is served at %s \n\n', + CONF.api.test_results_url.split('/#/')[0]) + return app diff --git a/requirements.txt b/requirements.txt index 96ff43c8..a7609559 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -SQLAlchemy==0.8.3 +SQLAlchemy>=0.8.3 alembic==0.5.0 #gunicorn 19.1.1 has a bug with threading module gunicorn==18 diff --git a/run-in-docker b/run-in-docker new file mode 100755 index 00000000..bc29c7bf --- /dev/null +++ b/run-in-docker @@ -0,0 +1,153 @@ +#!/bin/bash +TAG=$(BRANCH=$(git status -bs| grep "##" | awk '{print $2}'); echo ${BRANCH##*/}) +IMAGE="refstack:${TAG}" +CONTAINER="refstack_${TAG}" +PROJ_DIR=$(git rev-parse --show-toplevel) + +function usage () { +set +x +echo "Usage: $0 [OPTIONS] [COMMAND]" +echo "Build '${IMAGE}' image if it is does not exist." +echo "Run '${CONTAINER}' container and execute COMMAND in it." +echo "Default COMMAND is 'api-up'" +echo "If container '${CONTAINER}' exists (running or stopped) it will be reused." +echo "If you want to get access to your local Refstack not only from localhost, " +echo "please specify public Refstack host:port in env[REFSTACK_HOST]." +echo "You can customize Refstack API config by editing docker/refstack.conf.tmpl." +echo "It is bash template. You can use \${SOME_ENV_VARIABLE} in it." +echo "Default is 127.0.0.1:8000" +echo "" +echo " -r Force delete '${CONTAINER}' container and run it again." +echo " Main usecase for it - updating config from templates" +echo " -b Force delete '${IMAGE}' image and build it again" +echo " Main usecase for it - force build new python/js env" +echo " -i Run container with isolated MySQL data." +echo " By default MySQL data stores in refstack_data_DATA-BASE-REVISON container" +echo " It reuses if such container exists. If you want to drop DB data, just execute" +echo " sudo docker rm refstack_data_DATA-BASE-REVISON" +echo " -d Turn on debug information" +echo " -h Print this usage message" +echo "" +echo "" +echo "Using examples:" +echo "" +echo "Run Refstack API:" +echo "$ ./run-in-docker" +echo "" +echo "Run Refstack API by hands:" +echo "$ ./run-in-docker bash" +echo "$ activate" +echo "$ pecan serve refstack/api/config.py" +echo "" +echo "Open shell in container:" +echo "$ ./run-in-docker bash" +echo "" +echo "Open mysql console in container:" +echo "$ ./run-in-docker bash" +echo "$ mysql" +} + +build_image () { +sudo docker rm -f ${CONTAINER} +PREV_ID=$(sudo docker images refstack | grep ${TAG} | awk '{print $3}') +echo "Try to build ${IMAGE} image" +sudo docker build -t ${IMAGE} -f ${PROJ_DIR}/docker/Dockerfile ${PROJ_DIR} || exit $? +NEW_ID=$(sudo docker images refstack | grep ${TAG} | awk '{print $3}') +if [[ ${PREV_ID} ]] && [[ ! ${PREV_ID} == ${NEW_ID} ]]; then + sudo docker rmi -f ${PREV_ID} && echo "Previous image removed" +fi +} + +wait_ready() { +while true; do + echo "Wait while container is not ready" + sudo docker exec ${CONTAINER} [ ! -e /tmp/is-not-ready ] && \ + echo "Container ${CONTAINER} is running!" && break + sleep 1 +done +} + +run_container (){ +echo "Stop all other refstack containers" +for id in $(sudo docker ps -q); do + NAME=$(sudo docker inspect --format='{{.Name}}' $id) + if [[ ${NAME} == /refstack_* ]] && [[ ! ${NAME} == "/${CONTAINER}" ]]; then + echo "Stopped container ${NAME}" && sudo docker stop $id + fi +done +if [[ $(sudo docker ps -a | grep "${CONTAINER}") ]]; then + echo "Container ${CONTAINER} exists it is reused" + sudo docker start ${CONTAINER} + wait_ready +else + echo "Try to run container ${CONTAINER}" + sudo docker run -d \ + -e REFSTACK_HOST=${REFSTACK_HOST:-127.0.0.1} \ + -e DEBUG_MODE=${DEBUG_MODE} \ + -v ${PROJ_DIR}:/refstack:ro -p 443:443 --name ${CONTAINER} \ + ${IMAGE} start.sh -s + wait_ready + if [[ ! ${ISOLATED_DB} ]]; then + DB_VERSION=$(sudo docker exec -it ${CONTAINER} api-db-version) + DB_CONTAINER=refstack_data_${DB_VERSION::-1} + sudo docker rm -f ${CONTAINER} + if [[ ! $(sudo docker ps -a | grep "${DB_CONTAINER}") ]]; then + sudo docker run -v /home/dev/mysql --name ${DB_CONTAINER} ubuntu /bin/true + echo "Container with mysql data ${DB_CONTAINER} created" + sudo docker run -d \ + -e REFSTACK_HOST=${REFSTACK_HOST:-127.0.0.1} \ + -e DEBUG_MODE=${DEBUG_MODE} \ + -v ${PROJ_DIR}:/refstack:ro --volumes-from ${DB_CONTAINER} -p 443:443 \ + --name ${CONTAINER} ${IMAGE} + wait_ready + sudo docker exec ${CONTAINER} api-init-db + echo "DB init done" + else + sudo docker run -d \ + -e REFSTACK_HOST=${REFSTACK_HOST:-127.0.0.1} \ + -e DEBUG_MODE=${DEBUG_MODE} \ + -v ${PROJ_DIR}:/refstack:ro --volumes-from ${DB_CONTAINER} -p 443:443 \ + --name ${CONTAINER} ${IMAGE} + echo "Container with mysql data ${DB_CONTAINER} attached to ${CONTAINER}" + wait_ready + fi + + + fi +fi +} + +COMMAND="" +while [[ $1 ]] +do + case "$1" in + -h) usage + exit 0;; + -r) echo "Try to remove old ${CONTAINER} container" + sudo docker rm -f ${CONTAINER} + shift;; + -i) echo "Run container with isolated MySQL data." + echo "By default MySQL data stores in refstack_data_[DATA-BASE-REVISON] container" + echo "It reuses if such container exists. If you want to drop DB data, just execute" + echo "sudo docker rm ${DB_CONTAINER}" + ISOLATED_DB=true + shift;; + -b) FORCE_BUILD=true + shift;; + -d) DEBUG_MODE=true + shift;; + *) COMMAND="${COMMAND} $1" + shift;; + esac +done + +[[ ${DEBUG_MODE} ]] && set -x + +#Build proper image if it does not exist of force rebuild fired +if [[ ${FORCE_BUILD} ]] || [[ ! $(sudo docker images refstack | grep ${TAG}) ]]; then + build_image +fi +#Run or start(if it exists) proper container +[[ ! $(sudo docker ps | grep ${CONTAINER}) ]] && run_container + +sudo docker exec -it ${CONTAINER} ${COMMAND:-api-up} \ No newline at end of file