diff --git a/.gitignore b/.gitignore index c3fadaa5d6..2c803bfe6a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ _build/ .* !.gitignore !.mailmap +!.gitlab-ci.yml # compiled python files *.py[co] __pycache__/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 636b14c48f..abfc254791 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,33 +1,270 @@ -image: ubuntu:xenial stages: - - test -job_test_base_pg_96: - stage: test + - unittest + - build + - integrationtest + - builddev + - deploy + - package + +variables: + GITLAB_REGISTRY: "registry.gitlab.com/${CI_PROJECT_PATH}" + UBUNTU_BASE_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/base/ubuntu" + UBUNTU_FLECTRA_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/ubuntu" + UBUNTUDEV_FLECTRA_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/ubuntudev" + # DOCKER_HOST: tcp://docker:2375 + # DOCKER_DRIVER: overlay2 + + # Configure postgres service (https://hub.docker.com/_/postgres/) + POSTGRES_DB: postgres + POSTGRES_USER: flectra + POSTGRES_PASSWORD: flectra + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + +#-------------------------------------------------------------------- +# Job Templates +#-------------------------------------------------------------------- +.psql9:template: + services: &psql9_definition + - name: postgres:9-alpine + alias: postgres9 + +.psql10:template: + services: &psql10_definition + - name: postgres:10-alpine + alias: postgres10 + +.psql11:template: + services: &psql11_definition + - name: postgres:11-alpine + alias: postgres11 + +.unittest:template: &unittest_definition + image: ${UBUNTU_BASE_IMAGE}:latest + stage: unittest + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" + paths: + - flectra.log + when: always + expire_in: 2 week + coverage: '/^TOTAL.*\s+(\d+\%)$/' + only: + - branches + except: + - schedules + +.build:template: &build_definition + services: + - name: docker:dind + except: + - schedules + image: docker:stable + +.integrationtest:template: &integrationtest_definition + stage: integrationtest + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" + paths: + - flectra.log + when: always + expire_in: 2 week + only: + - tags + +.build:base:template: &build_base_definition + stage: build + services: + - name: docker:dind + only: + - schedules + image: docker:stable + +#-------------------------------------------------------------------- +# Jobs of stage unittest +#-------------------------------------------------------------------- +test:pg9_base: + <<: *unittest_definition + services: *psql9_definition script: - - apt-get update && apt-get install -y apt-utils locales git - - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} - - gitlab_ci_install_flectra - - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=base" -job_test_all_96: - stage: test + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres9" + +test:pg9_all: + <<: *unittest_definition + services: *psql9_definition script: - - apt-get update && apt-get install -y apt-utils locales git - - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} - - gitlab_ci_install_flectra - - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=all" -job_test_base_100: - stage: test + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres9" + +test:pg10_base: + <<: *unittest_definition + services: *psql10_definition script: - - apt-get update && apt-get install -y apt-utils locales git - - export POSTGRESQL_VERSION=10 - - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} - - gitlab_ci_install_flectra - - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=base" -job_test_all_100: - stage: test + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres10" + +test:pg10_all: + <<: *unittest_definition + services: *psql10_definition script: - - apt-get update && apt-get install -y apt-utils locales git - - export POSTGRESQL_VERSION=10 - - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} - - gitlab_ci_install_flectra - - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=all" \ No newline at end of file + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres10" + +test:pg11_base: + <<: *unittest_definition + services: *psql11_definition + script: + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres11" + +test:pg11_all: + <<: *unittest_definition + services: *psql11_definition + script: + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres11" + +#-------------------------------------------------------------------- +# Jobs of stage build +#-------------------------------------------------------------------- +build:flectra_ubuntu: + <<: *build_definition + stage: build + only: + - tags + script: + - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker pull ${UBUNTU_FLECTRA_IMAGE}:latest || true + - docker build --cache-from ${UBUNTU_FLECTRA_IMAGE}:latest + --build-arg BASE_IMAGE=${UBUNTU_BASE_IMAGE} + --build-arg GITLAB_OWNER=${CI_PROJECT_NAMESPACE} + --build-arg GITLAB_COMMIT=${CI_COMMIT_SHA} + --file ./setup/docker/flectra/ubuntu/Dockerfile + --tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + ./setup/docker/ + - docker push ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + +#-------------------------------------------------------------------- +# Jobs of stage integrationtest +#-------------------------------------------------------------------- +itest:ubuntu_pg9_all: + <<: *integrationtest_definition + services: *psql9_definition + image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + script: + - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres9 + +itest:ubuntu_pg10_all: + <<: *integrationtest_definition + services: *psql10_definition + image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + script: + - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres10 + +itest:ubuntu_pg11_all: + <<: *integrationtest_definition + services: *psql11_definition + image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + script: + - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres11 + +#-------------------------------------------------------------------- +# Jobs of stage builddev +#-------------------------------------------------------------------- +build:flectra_ubuntu_dev: + <<: *build_definition + stage: builddev + only: + - tags + script: + - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA || true + - docker build --cache-from ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + --build-arg BASE_IMAGE=${UBUNTU_FLECTRA_IMAGE} + --build-arg BASE_TAG=${CI_COMMIT_SHA} + --file ./setup/docker/flectra/dev/Dockerfile + --tag ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA + ./setup/docker/ + - docker push ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA + +#-------------------------------------------------------------------- +# Jobs of stage deploy +#-------------------------------------------------------------------- +deploy:flectra_gitlab: + <<: *build_definition + stage: deploy + only: + - tags + script: + - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_TAG + - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTU_FLECTRA_IMAGE}:latest + - docker push ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_TAG + - docker push ${UBUNTU_FLECTRA_IMAGE}:latest + +deploy:flectradev_gitlab: + <<: *build_definition + stage: deploy + only: + - tags + script: + - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker pull ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA + - docker tag ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_TAG + - docker tag ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTUDEV_FLECTRA_IMAGE}:latest + - docker push ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_TAG + - docker push ${UBUNTUDEV_FLECTRA_IMAGE}:latest + +deploy:flectra_dockerhub: + <<: *build_definition + stage: deploy + only: + - tags + script: + - echo -n $DOCKER_HUB_PASSWORD | docker login -u $DOCKER_HUB_USER --password-stdin https://index.docker.io/v1/ + - docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${DOCKER_HUB_ROOT}/flectra:$CI_COMMIT_TAG + - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${DOCKER_HUB_ROOT}/flectra:latest + - docker push ${DOCKER_HUB_ROOT}/flectra:$CI_COMMIT_TAG + - docker push ${DOCKER_HUB_ROOT}/flectra:latest + +.deploy:flectra_bin: + services: + - name: docker:dind + - name: postgres:10-alpine + alias: postgres + only: + - tags + image: docker:stable + stage: package + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" + paths: + - build.log + when: always + expire_in: 1 day + script: + - apk add python3 python3-dev py-pip build-base rsync dpkg-dev rpm + - pip3 install pexpect + - mkdir /opt/flectra/ + - mkdir /opt/flectra/log + - mkdir /opt/flectra/build + - mkdir /opt/flectra/public + - cd ${CI_PROJECT_DIR}/setup && python3 ./package.py -b /opt/flectra/build -p /opt/flectra/public --no-windows > ${CI_PROJECT_DIR}/build.log + - find /opt/flectra/public -maxdepth 3 + + +#-------------------------------------------------------------------- +# Scheduled jobs of stage build (executed by gitlab scheduler) +#-------------------------------------------------------------------- + +build:base_ubuntu: + <<: *build_base_definition + script: + - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker pull ${UBUNTU_BASE_IMAGE}:latest || true + - docker build --cache-from ${UBUNTU_BASE_IMAGE}:latest -f ./setup/docker/base/ubuntu/Dockerfile ./setup/docker/ --tag ${UBUNTU_BASE_IMAGE}:latest + - docker push ${UBUNTU_BASE_IMAGE}:latest + diff --git a/setup/docker/QUESTIONS.md b/setup/docker/QUESTIONS.md new file mode 100644 index 0000000000..0bdd0ef0de --- /dev/null +++ b/setup/docker/QUESTIONS.md @@ -0,0 +1,4 @@ +Questions about docker integration +================================== + +- how to avoid duplicate requirements.txt (e.g. setup/docker/requirements.txt) \ No newline at end of file diff --git a/setup/docker/README.md b/setup/docker/README.md new file mode 100644 index 0000000000..6f9629d782 --- /dev/null +++ b/setup/docker/README.md @@ -0,0 +1,94 @@ +Continuous Integration and Docker Image Generation +================================================== + + +Intro +----- + +This directory contains all the tools needed for optimized continuous integration testing +and direct deployment of docker images for development, test and production use. + +On every push or merge into core flectra repository some gitlab-ci jobs will be executed +to test the changes and generate new docker images. The details are described in the +following chapters. + +Goal +---- + +All code changes should be tested automatically in an acceptable duration and the revision +should be available with a few steps to run it as a docker container for manual testing. + +Docker Image Inheritance +------------------------ + +There are some base docker images which are used for better performance +(and less waiting time for developers). Based on that images other images are generated. + +The following image are deployed in gitlab registry for flectra: + +| Image | Tag | Base Image | Trigger | Content +|--------------|-------------|--------------------|-----------|------------ +| base/ubuntu | latest | ubuntu:xenial | scheduler | all required linux and python package to run flectra (ubuntu based) +| base/debian | latest | debian:jessie | scheduler | all required linux and python package to run flectra (debian based) +| ubuntu | **_build_** | base/ubuntu:latest | branch | full flectra installation and can be used for runbot like testing (ubuntu based) +| debian | **_build_** | base/debian:latest | branch | full flectra installation and can be used for runbot like testing (debian based) +| ubuntu | **_tag_** | base/ubuntu:latest | tag | full flectra installation for use in production (ubuntu based) +| debian | **_tag_** | base/debian:latest | tag | full flectra installation for use in production (debian based) + +some additional images for testing and building are used as services: + +- **postgres:9-alpine**: latest PostgreSQL 9.x image for running tests +- **postgres:10-alpine**: latest PostgreSQL 10.x image for running tests +- **docker:dind**: Special image containing docker to build and push images + +Stages +------ + +The whole CI and docker process is split in four parts (called stages): + +1. unittest: the jobs of this stage will validate the changes +2. build: at this stage the docker images are generated +3. integrationtest: these jobs will validate the flectra installation of docker images generated in previous stage +4. deploy: these jobs will deploy the docker images with flectra for production use (tagged wit git tag and as latest) + +Only when all jobs of one stage are successful done, the next stage is started. + +Jobs +---- + +### Stage: unittest + +The following jobs are started on every push or merge of code: + +- **test:pg9_base**: Run test of base module against PostgreSQL 9.x database +- **test:pg9_all**: Run test of all modules against PostgreSQL 9.x database +- **test:pg10_base**: Run test of base module against PostgreSQL 10.x database +- **test:pg9_all**: Run test of all modules against PostgreSQL 10.x database + +### Stage: build + +The following jobs are only started by gitlab scheduler: + +-**build:base_ubuntu**: Build and push base ubuntu image containing all linux and python requirements +-**build:base_debian**: Build and push base debian image containing all linux and python requirements + +The following jobs are started on push for a branch: + +-**build:flectra_ubuntu**: Build and push image with installed flectra, tagged with commit id (ubuntu based) +-**build:flectra_debian**: Build and push image with installed flectra, tagged with commit id (debian based) + +### Stage: integrationtest + +The following jobs are started on push for a branch: + +- **itest:ubuntu_pg9**: Run test of all modules against PostgreSQL 9.x database with flectra included in docker image (ubuntu based) +- **itest:ubuntu_pg10**: Run test of all modules against PostgreSQL 10.x database with flectra included in docker image (ubuntu based) +- **itest:debian_pg9**: Run test of all modules against PostgreSQL 9.x database with flectra included in docker image (debian based) +- **itest:debian_pg10**: Run test of all modules against PostgreSQL 10.x database with flectra included in docker image (debian based) + +### Stage: deploy + +The following jobs are started on push for tag: + +- **deploy:flectra_ubuntu**: Generate docker image tagged with git tag (=flectra version) and as latest (ubuntu based) +- **deploy:flectra_debian**: Generate docker image tagged with git tag (=flectra version) and as latest (debian based) diff --git a/setup/docker/base/debian/Dockerfile b/setup/docker/base/debian/Dockerfile new file mode 100644 index 0000000000..d83c225275 --- /dev/null +++ b/setup/docker/base/debian/Dockerfile @@ -0,0 +1,62 @@ +FROM debian:jessie + +LABEL maintainer="FlectraHQ " +LABEL authors="FlectraHQ,Jamotion" + +ARG POSTGRESQL_VERSION=9.6 + +RUN apt-get update && \ + apt-get install wget -y + +# adding postgresql debian repo in order to access all available postgresql version +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" > /etc/apt/sources.list.d/pgdg.list + +RUN apt-get update && \ + apt-get install git gnupg2 python3 curl expect-dev node-less \ + python3-pillow python3-lxml python-ldap3 python3-dev python3-pip python3-setuptools \ + libpq5 libpq-dev postgresql-${POSTGRESQL_VERSION} postgresql-client-${POSTGRESQL_VERSION} \ + postgresql-contrib-${POSTGRESQL_VERSION} postgresql-server-dev-${POSTGRESQL_VERSION} \ + fontconfig libxfont1 xfonts-encodings xfonts-utils xfonts-base xfonts-75dpi \ + npm nodejs libldap2-dev libsasl2-dev libxml2-dev libxslt1-dev libjpeg-dev zlib1g-dev libfontconfig1 -y + +RUN locale-gen --purge en_US.UTF-8 && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:UTF-8"\n' > /etc/default/locale + +ADD requirements.txt /tmp/requirements.txt + +RUN pip3 install --upgrade pip +RUN pip3 install passlib +RUN pip3 install -r /tmp/requirements.txt +RUN npm install -g less less-plugin-clean-css -y && \ + ln -s /usr/bin/nodejs /usr/bin/node + +RUN curl -o wkhtmltox.tar.xz -SL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \ + && echo '3f923f425d345940089e44c1466f6408b9619562 wkhtmltox.tar.xz' | sha1sum -c - \ + && tar xvf wkhtmltox.tar.xz \ + && cp wkhtmltox/lib/* /usr/local/lib/ \ + && cp wkhtmltox/bin/* /usr/local/bin/ \ + && cp -r wkhtmltox/share/man/man1 /usr/local/share/man/ + + +# install CI tools +RUN pip3 install coverage coveralls codecov simplejson pyserial pyyaml unittest2 pyopenssl>=16.2.0 && \ + wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz && \ + gunzip GeoLiteCity.dat.gz && \ + mkdir -p /usr/share/GeoIP/ && \ + mv GeoLiteCity.dat /usr/share/GeoIP/ + +RUN addgroup --gid=5500 flectra && adduser --system --uid=5500 --gid=5500 --home=/opt/flectra --shell=/bin/bash flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER root \ No newline at end of file diff --git a/setup/docker/base/ubuntu/Dockerfile b/setup/docker/base/ubuntu/Dockerfile new file mode 100644 index 0000000000..5265e97071 --- /dev/null +++ b/setup/docker/base/ubuntu/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:bionic + +LABEL maintainer="FlectraHQ " +LABEL authors="FlectraHQ,Jamotion" + +ARG POSTGRESQL_VERSION=10 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install wget -y + +RUN apt-get update && \ + apt-get install git gnupg2 python3 curl expect-dev node-less \ + python3-pillow python3-lxml python-ldap3 python3-dev python3-pip python3-setuptools python3-passlib \ + postgresql-${POSTGRESQL_VERSION} postgresql-server-dev-${POSTGRESQL_VERSION} \ + fontconfig libxfont2 xfonts-encodings xfonts-utils xfonts-base xfonts-75dpi \ + npm nodejs libldap2-dev libsasl2-dev libxml2-dev libxslt1-dev libjpeg-dev zlib1g-dev libfontconfig1 libmagickwand-dev -y + +RUN locale-gen --purge en_US.UTF-8 && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:UTF-8"\n' > /etc/default/locale + +ADD requirements.txt /tmp/requirements.txt + +RUN pip3 install --upgrade pip +RUN pip3 install -r /tmp/requirements.txt +RUN npm install -g less less-plugin-clean-css -y + +RUN curl -SsLO https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb && \ + apt-get install -f ./wkhtmltox_0.12.5-1.bionic_amd64.deb + +# install CI tools +RUN pip3 install coverage coveralls codecov simplejson pyserial pyyaml unittest2 pyopenssl>=16.2.0 \ + validate_email zpl2 unidecode shapely-geojson simplejson shapely geojson unidecode \ + email_validator wand imageio moviepy + +RUN addgroup --gid=5500 flectra && adduser --system --uid=5500 --gid=5500 --home=/opt/flectra --shell=/bin/bash flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER root \ No newline at end of file diff --git a/setup/docker/entrypoint.sh b/setup/docker/entrypoint.sh new file mode 100755 index 0000000000..01f388d275 --- /dev/null +++ b/setup/docker/entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e + +# set the postgres database host, port, user and password according to the environment +# and pass them as arguments to the flectra process if not present in the config file +: ${HOST:=${DB_PORT_5432_TCP_ADDR:='db'}} +: ${PORT:=${DB_PORT_5432_TCP_PORT:=5432}} +: ${USER:=${DB_ENV_POSTGRES_USER:=${POSTGRES_USER:='flectra'}}} +: ${PASSWORD:=${DB_ENV_POSTGRES_PASSWORD:=${POSTGRES_PASSWORD:='flectra'}}} + +DB_ARGS=() +function check_config() { + param="$1" + value="$2" + DB_ARGS+=("--${param}") + DB_ARGS+=("${value}") +} +if [[ -z "${IGNORE_ENV}" ]]; then + check_config "db_host" "$HOST" + check_config "db_port" "$PORT" + check_config "db_user" "$USER" + check_config "db_password" "$PASSWORD" +fi + +case "$1" in + -- | flectra) + shift + if [[ "$1" == "scaffold" ]] ; then + exec /opt/flectra/flectra-bin "$@" + else + exec /opt/flectra/flectra-bin "$@" "${DB_ARGS[@]}" + fi + ;; + -*) + exec /opt/flectra/flectra-bin "$@" "${DB_ARGS[@]}" + ;; + *) + exec "$@" +esac + +exit 1 diff --git a/setup/docker/flectra.conf b/setup/docker/flectra.conf new file mode 100644 index 0000000000..b7b571fd77 --- /dev/null +++ b/setup/docker/flectra.conf @@ -0,0 +1,37 @@ +[options] +addons_path = /mnt/extra-addons +data_dir = /var/lib/flectra +; admin_passwd = admin +; csv_internal_sep = , +; db_maxconn = 64 +; db_name = False +; db_template = template1 +; dbfilter = .* +; debug_mode = False +; email_from = False +; limit_memory_hard = 2684354560 +; limit_memory_soft = 2147483648 +; limit_request = 8192 +; limit_time_cpu = 60 +; limit_time_real = 120 +; list_db = True +; log_db = False +; log_handler = [':INFO'] +; log_level = info +; logfile = None +; longpolling_port = 7072 +; max_cron_threads = 2 +; osv_memory_age_limit = 1.0 +; osv_memory_count_limit = False +; smtp_password = False +; smtp_port = 25 +; smtp_server = localhost +; smtp_ssl = False +; smtp_user = False +; workers = 0 +; xmlrpc = True +; xmlrpc_interface = +; xmlrpc_port = 7073 +; xmlrpcs = True +; xmlrpcs_interface = +; xmlrpcs_port = 8071 diff --git a/setup/docker/flectra/dev/Dockerfile b/setup/docker/flectra/dev/Dockerfile new file mode 100644 index 0000000000..556077b5d3 --- /dev/null +++ b/setup/docker/flectra/dev/Dockerfile @@ -0,0 +1,35 @@ +#Partially from Odoo's dockerfile +#github.com/odoo/docker + +ARG BASE_IMAGE +ARG BASE_TAG +FROM ${BASE_IMAGE}:${BASE_TAG} +LABEL maintainer="FlectraHQ " +LABEL authors="FlectraHQ,Jamotion" + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER root + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +# Install and configure OpenSSH Server (for development mode only) +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install openssh-server pwgen build-essential +RUN mkdir -p /var/run/sshd \ + && sed -i "s/UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config \ + && sed -i "s/UsePAM.*/UsePAM no/g" /etc/ssh/sshd_config \ + && sed -i "s/PermitRootLogin.*/PermitRootLogin yes/g" /etc/ssh/sshd_config + +# add special scripts for development +ADD flectra/dev/bin /usr/local/bin + +VOLUME ["/opt/flectra", "/etc/flectra", "/var/log/flectra"] + +# Expose Flectra services +EXPOSE 7072 7073 22 + +ENTRYPOINT ["/usr/local/bin/run.sh"] diff --git a/setup/docker/flectra/dev/bin/run.sh b/setup/docker/flectra/dev/bin/run.sh new file mode 100644 index 0000000000..1e4ff76be5 --- /dev/null +++ b/setup/docker/flectra/dev/bin/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +if [ "${AUTHORIZED_KEYS}" != "**None**" ]; then + echo "=> Found authorized keys" + mkdir -p /root/.ssh + chmod 700 /root/.ssh + touch /root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys + IFS=$'\n' + arr=$(echo ${AUTHORIZED_KEYS} | tr "," "\n") + for x in $arr + do + x=$(echo $x |sed -e 's/^ *//' -e 's/ *$//') + cat /root/.ssh/authorized_keys | grep "$x" >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "=> Adding public key to /root/.ssh/authorized_keys: $x" + echo "$x" >> /root/.ssh/authorized_keys + fi + done +fi + +/usr/local/bin/set_root_pw.sh +exec /usr/sbin/sshd -D \ No newline at end of file diff --git a/setup/docker/flectra/dev/bin/set_root_pw.sh b/setup/docker/flectra/dev/bin/set_root_pw.sh new file mode 100644 index 0000000000..c6eb1a20b7 --- /dev/null +++ b/setup/docker/flectra/dev/bin/set_root_pw.sh @@ -0,0 +1,25 @@ +#!/bin/bash +if [ -f /etc/flectra/ssh-pwd ]; then + echo "Root password already saved. Using this one" + PASS=$( Setting a ${_word} password to the root user" +fi + +echo "root:$PASS" | chpasswd +echo "flectra:$PASS" | chpasswd +echo "=> Done!" +echo $PASS > /etc/flectra/ssh-pwd +IP=$(hostname --ip-address) +echo "========================================================================" +echo " You can now connect to this Developer Container via SSH using:" +echo "" +echo " ssh -p root@${IP}" +echo " or" +echo " ssh -p flectra@${IP}" +echo "" +echo " and enter the password '$PASS' when prompted" +echo "" +echo "========================================================================" \ No newline at end of file diff --git a/setup/docker/flectra/ubuntu/Dockerfile b/setup/docker/flectra/ubuntu/Dockerfile new file mode 100644 index 0000000000..d8c820b233 --- /dev/null +++ b/setup/docker/flectra/ubuntu/Dockerfile @@ -0,0 +1,56 @@ +#Partially from Odoo's dockerfile +#github.com/odoo/docker + +ARG BASE_IMAGE +FROM ${BASE_IMAGE}:latest +LABEL maintainer="FlectraHQ " +LABEL authors="FlectraHQ,Jamotion" + +ARG GITLAB_OWNER=flectra +ARG GITLAB_COMMIT=master +ARG ARG_DB_HOST=postgres +ARG ARG_DB_PORT=5432 +ARG ARG_DB_USER=flectra +ARG ARG_DB_PASSWORD=flectra + +ENV BRANCH=$ARG_BRANCH +ENV HOST=$ARG_DB_HOST +ENV PORT=$ARG_DB_PORT +ENV USER=$ARG_DB_USER +ENV PASSWORD=$ARG_DB_PASSWORD + +USER root +RUN mkdir -p /var/log/flectra && \ + chown -R flectra:flectra /var/log/flectra/ + +RUN locale-gen --purge en_US.UTF-8 && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:en"\n' > /etc/default/locale + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +RUN cd /opt/flectra && \ + wget -q -O flectra.tar.gz https://gitlab.com/${GITLAB_OWNER}/flectra/-/archive/${GITLAB_COMMIT}/flectra.tar.gz && \ + tar -zxvf flectra.tar.gz flectra-${GITLAB_COMMIT}/flectra-bin flectra-${GITLAB_COMMIT}/flectra flectra-${GITLAB_COMMIT}/addons && \ + cd flectra-${GITLAB_COMMIT} && \ + mv addons/* flectra/addons/ && \ + mv flectra ../. && \ + mv flectra-bin ../. && \ + cd /opt/flectra && \ + rm -rf flectra-${GITLAB_COMMIT} + +EXPOSE 7073 7072 + +ADD entrypoint.sh /opt/flectra/entrypoint.sh + +ENTRYPOINT ["/opt/flectra/entrypoint.sh"] +CMD ["flectra"] + diff --git a/setup/docker/gitlab-ci/gitlab_after_sucesss b/setup/docker/gitlab-ci/gitlab_after_sucesss new file mode 100755 index 0000000000..3a36aa2bba --- /dev/null +++ b/setup/docker/gitlab-ci/gitlab_after_sucesss @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +# Part of code taken from github.com/OCA/maintainer-quality-tools + +import os + +from coverage.cmdline import main as coverage_main +from coveralls import cli as coveralls_cli +from codecov import main as codecov_main + + +coverage_main(["report", "--show-missing"]) +try: + coveralls_cli.main(argv=None) +except: + pass +try: + codecov_main(argv=None) +except: + pass diff --git a/setup/docker/gitlab-ci/gitlab_test_flectra.py b/setup/docker/gitlab-ci/gitlab_test_flectra.py new file mode 100755 index 0000000000..316fe95183 --- /dev/null +++ b/setup/docker/gitlab-ci/gitlab_test_flectra.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 + +# Part of code taken from github.com/OCA/maintainer-quality-tools + +import re +import os +import shutil +import subprocess +import sys +import uuid +import argparse +from coverage.cmdline import main as coverage_main +import locale + +try: + basestring +except NameError: + basestring = str + +RED = "\033[1;31m" +GREEN = "\033[1;32m" +YELLOW = "\033[1;33m" +YELLOW_LIGHT = "\033[33m" +CLEAR = "\033[0;m" + +# modules that are either added as dependencies for other modules, so better to ignore them for the moment +modules_to_ignore = ["account_analytic_default", "account_bank_statement_import", "account_budget", + "account_cancel", "account_check_printing", "account_payment", "account_tax_python", + "account_test", "account_voucher", "analytic", "anonymization", "association", + "auth_crypt", "auth_ldap", "auth_oauth", "auth_signup", "barcodes", "base_address_city", + "base_address_extended", "base_automation", "base_gengo", "base_geolocalize", "base_iban", + "base_import", "base_import_module", "base_setup", "base_sparse_field", "base_vat", + "base_vat_autocomplete", "bus", "calendar_sms", "crm_livechat", + "crm_phone_validation", "crm_project", "decimal_precision", "delivery", "document", "event", + "event_sale", "fetchmail", "gamification", "gamification_sale_crm", "google_account", + "google_calendar", "google_drive", "google_spreadsheet", + "hr_contract", "hr_expense_check", "hr_gamification", "hr_maintenance", + "hr_org_chart", "hr_payroll", "hr_payroll_account", "hr_recruitment_survey", + "hr_timesheet_attendance", "http_routing", "hw_blackbox_be", "hw_escpos", "hw_posbox_homepage", + "hw_posbox_upgrade", "hw_proxy", "hw_scale", "hw_scanner", "hw_screen", "link_tracker", + "live_currency_rate", "mass_mailing_event", "mass_mailing_event_track", "membership", + "mrp_byproduct", "note_pad", "pad", "pad_project", "password_security", + "payment", "payment_adyen", "payment_authorize", "payment_buckaroo", "payment_ogone", + "payment_paypal", "payment_payumoney", "payment_sips", "payment_stripe", "payment_transfer", + "phone_validation", "portal", "pos_cache", "pos_data_drinks", "pos_discount", "pos_mercury", + "pos_reprint", "pos_restaurant", "pos_sale", "procurement_jit", "product", "product_email_template", + "product_expiry", "product_extended", "product_margin", "project_timesheet_holidays", "rating", + "rating_project", "report_intrastat", "resource", "sale_mrp", "sale_order_dates", + "sale_payment", "sale_service_rating", "sale_timesheet", "sales_team", "sms", "stock_dropshipping", + "stock_landed_costs", "stock_picking_batch", "survey_crm", "theme_bootswatch", "theme_default", + "transifex", "utm", "web", "web_diagram", "web_editor", "web_kanban_gauge", "web_planner", + "web_settings_dashboard", "web_tour", "website_crm", "website_crm_partner_assign", + "website_crm_phone_validation", "website_customer", "website_event_questions", + "website_event_sale", "website_event_track", "website_form", "website_form_project", + "website_forum_doc", "website_gengo" , "website_google_map", "website_hr", "website_hr_recruitment", + "website_links", "website_livechat", "website_mail", "website_mail_channel", "website_mass_mailing", + "website_membership", "website_partner", "website_payment", "website_quote", "website_rating", + "website_rating_project", "website_sale_comparison", "website_sale_delivery", "website_sale_digital", + "website_sale_management", "website_sale_options", "website_sale_stock", "website_sale_stock_options", + "website_sale_wishlist", "website_theme_install", "website_twitter", "theme_art", "theme_hermit", + "theme_leith", "theme_techreceptives"] + + +def colorized(text, color): + return '\n'.join( + map(lambda line: color + line + CLEAR, text.split('\n'))) + + +def green(text): + return colorized(text, GREEN) + + +def yellow(text): + return colorized(text, YELLOW) + + +def red(text): + return colorized(text, RED) + + +def yellow_light(text): + return colorized(text, YELLOW_LIGHT) + + +fail_msg = red("FAIL") +success_msg = green("Success") + + +def has_test_errors(fname, dbname, check_loaded=True): + """ + Check a list of log lines for test errors. + Extension point to detect false positives. + """ + # Rules defining checks to perform + # this can be + # - a string which will be checked in a simple substring match + # - a regex object that will be matched against the whole message + # - a callable that receives a dictionary of the form + # { + # 'loglevel': ..., + # 'message': ...., + # } + errors_ignore = [ + 'Mail delivery failed', + 'failed sending mail', + ] + errors_report = [ + lambda x: 'loglevel' in x and x['loglevel'] == 'CRITICAL', + 'At least one test failed', + 'no access rules, consider adding one', + 'invalid module names, ignored', + ] + + def make_pattern_list_callable(pattern_list): + for i in range(len(pattern_list)): + if isinstance(pattern_list[i], basestring): + regex = re.compile(pattern_list[i]) + pattern_list[i] = lambda x, regex=regex: \ + regex.search(x['message']) + elif hasattr(pattern_list[i], 'match'): + regex = pattern_list[i] + pattern_list[i] = lambda x, regex=regex: \ + regex.search(x['message']) + + make_pattern_list_callable(errors_ignore) + make_pattern_list_callable(errors_report) + + print("-" * 10) + # Read log file removing ASCII color escapes: + # http://serverfault.com/questions/71285 + color_regex = re.compile(r'\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]') + log_start_regex = re.compile(r'^.*(?P(INFO|WARNING|DEBUG|ERROR|CRITICAL)).*?: (?P.*\S)\s*$') + log_records = [] + last_log_record = dict.fromkeys(log_start_regex.groupindex.keys()) + with open(fname, encoding='utf-8') as log: + for line in log: + line = color_regex.sub('', line) + if sys.stdout.encoding != 'UTF-8': + line = line.encode('ascii', 'ignore').decode('ascii') + match = log_start_regex.match(line) + if match: + last_log_record = match.groupdict() + log_records.append(last_log_record) + else: + last_log_record['message'] = '%s\n%s' % ( + last_log_record['message'], line.rstrip('\n') + ) + errors = [] + for log_record in log_records: + ignore = False + for ignore_pattern in errors_ignore: + if ignore_pattern(log_record): + ignore = True + break + if ignore: + break + print('{0}: {1}'.format(log_record['loglevel'], log_record['message'])) + for report_pattern in errors_report: + if report_pattern(log_record): + errors.append(log_record) + break + + if check_loaded: + if not [r for r in log_records if 'Modules loaded.' in r['message']]: + errors.append({'message': "Message not found: 'Modules loaded.'"}) + + if errors: + for e in errors: + print(e['message']) + print("-"*10) + return len(errors) + + +def run_flectra(type_of_db, server_path, host, port, user, password): + print("\n creating database for : ", type_of_db) + try: + db = str(uuid.uuid1()) + "-" + type_of_db + + # Setting locale is needed because of gitlab runner server have + # other locales + os.environ["LANG"] = "en_US.UTF-8" + os.environ["LANGUAGE"] = "en_US:en" + os.environ["LC_ALL"] = "en_US.UTF-8" + + os.environ["PGPASSWORD"] = password + subprocess.check_call( + ["createdb", "-U", user, "-h", host, "-p", port, "-T", + "template1", db]) + except subprocess.CalledProcessError: + print("Problem in creating database.") + else: + log_file = os.path.join(os.getcwd(), "flectra.log") + + # ugly hack as -i all is broken and does not take all modules into consideration + # need to remove l10n_* modules from the list as installing them all together + # creates trouble and build always fails + addons_path = os.path.join(server_path, "addons") + if not os.path.isdir(addons_path): + # Assume we are at integration test level where we can not test performance and client modules + addons_path = os.path.join(server_path, "flectra", "addons") + modules_to_ignore.append('test_performance') + modules_to_ignore.append('website') + + addons = list( + set([addon.name for addon in os.scandir(addons_path) if addon.is_dir() and "10n" not in addon.name]) - set( + modules_to_ignore)) + + if type_of_db == "all": + modules_to_init = str(",".join(addons)) + else: + modules_to_init = "base" + + cmd_flectra = ["unbuffer", "coverage", "run", "--source", server_path] + cmd_flectra += ["%s/flectra-bin" % (server_path), + "--db_host", + host, + "--db_port", + port, + "--db_user", + user, + "--db_password", + password, + "-d", + db, + '--db-filter', + db, + "--addons-path", + addons_path, + "--log-level=info", + "--stop-after-init", + "--test-enable", + "--init", modules_to_init] + print("CMD EXECUTED --->>> ", " ".join(cmd_flectra)) + + with open(log_file, 'w') as outfile: + with subprocess.Popen( + cmd_flectra, + stdout=outfile, + bufsize=1, + universal_newlines=True) as p: + returncode = p.wait() + outfile.close() + errors = has_test_errors(os.path.join(server_path, log_file), db) + return {'errors': errors, 'returncode': returncode} + + +def main(argv=None): + parser = argparse.ArgumentParser(description='Create Docker Instances....') + parser.add_argument('--build', dest='build', help='Port For Instance', required=True, default="base") + parser.add_argument('--server-path', dest='path', help='Flectra Path') + parser.add_argument('--host', dest='host', + help='PostgreSQL Host', + default="postgres") + parser.add_argument('--port', dest='port', + help='PostgreSQL Port', + default="5432") + parser.add_argument('--user', dest='user', + help='PostgreSQL user', + default="flectra") + parser.add_argument('--password', dest='password', + help='PostgreSQL password', + default="flectra") + args = parser.parse_args() + build = args.build + + if argv is None: + argv = sys.argv + res = {} + server_path = args.path or os.getcwd() + + if build == "all": + res['all'] = run_flectra( + type_of_db="all", + server_path=server_path, + host=args.host, + port=args.port, + user=args.user, + password=args.password, + ) + errors = 'all' in res and res['all'] and 'errors' in res['all'] and \ + res['all']['errors'] + + if build == "base": + res['base'] = run_flectra( + type_of_db="base", + server_path=server_path, + host=args.host, + port=args.port, + user=args.user, + password=args.password, + ) + errors = 'base' in res and res['base'] and 'errors' in res['base'] and \ + res['base']['errors'] + + print("************ ERRORS ************ : ", errors) + return errors + + +if __name__ == '__main__': + rc = main() + + if rc: + exit(rc) + + coverage_main(["report"]) + + exit(rc) diff --git a/setup/docker/requirements.txt b/setup/docker/requirements.txt new file mode 100644 index 0000000000..307eb7ba78 --- /dev/null +++ b/setup/docker/requirements.txt @@ -0,0 +1,49 @@ +Babel==2.3.4 +decorator==4.0.10 +docutils==0.12 +ebaysdk==2.1.5 +feedparser==5.2.1 +gevent==1.1.2 ; sys_platform != 'win32' and python_version < '3.7' +gevent==1.3.4 ; sys_platform != 'win32' and python_version >= '3.7' +greenlet==0.4.10 ; python_version < '3.7' +greenlet==0.4.13 ; python_version >= '3.7' +html2text==2016.9.19 +Jinja2==2.8.1 +lxml==3.7.1 ; sys_platform != 'win32' and python_version < '3.7' +lxml==4.2.3 ; sys_platform != 'win32' and python_version >= '3.7' +lxml ; sys_platform == 'win32' +Mako==1.0.4 +MarkupSafe==0.23 +mock==2.0.0 +num2words==0.5.6 +oauthlib==2.0.7 +ofxparse==0.16 +passlib==1.6.5 +phonenumbers==8.8.9 +Pillow==4.0.0 +psutil==4.3.1; sys_platform != 'win32' +psycopg2==2.7.3.1; sys_platform != 'win32' +pydot==1.2.3 +pyldap==2.4.28; sys_platform != 'win32' +pyparsing==2.1.10 +PyPDF2==1.26.0 +pyserial==3.1.1 +python-dateutil==2.5.3 +pytz==2016.7 +pyusb==1.0.0 +PyYAML==3.12 ; python_version < '3.7' +PyYAML==3.13 ; python_version >= '3.7' +qrcode==5.3 +reportlab==3.3.0 +requests==2.20.0 +suds-jurko==0.6 +vatnumber==1.2 +vobject==0.9.3 +Werkzeug==0.11.15 +XlsxWriter==0.9.3 +xlwt==1.3.* +xlrd==1.0.0 +unittest2==1.1.0 +numpy==1.14.3 +pypiwin32 ; sys_platform == 'win32' +pycrypto==2.6.1 \ No newline at end of file diff --git a/setup/package.py b/setup/package.py index ea9dcad0aa..4427f25fd3 100644 --- a/setup/package.py +++ b/setup/package.py @@ -362,13 +362,13 @@ def test_rpm(o): with docker('flectra-%s-fedora-nightly-tests' % docker_version, o.build_dir, o.pub) as fedora24: fedora24.release = '*.noarch.rpm' # Start postgresql - fedora24.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"') - fedora24.system('sleep 5') - fedora24.system('su postgres -c "createdb mycompany"') + # fedora24.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"') + # fedora24.system('sleep 5') + # fedora24.system('su postgres -c "createdb mycompany"') # Flectra install fedora24.system('dnf install -d 0 -e 0 /opt/release/%s -y' % fedora24.release) - fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -i base --stop-after-init"') - fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany &"') + fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -i base --stop-after-init -r flectra -w flectra --db_host postgres --db_port 5432"') + fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -r flectra -w flectra --db_host postgres --db_port 5432 &"') def test_exe(o): logging.info('Testng windows installer in KVM')