From e74417f94487fd19565327372c47bbf2365023bf Mon Sep 17 00:00:00 2001 From: Renzo Meister Date: Tue, 31 Jul 2018 17:31:29 +0200 Subject: [PATCH] [IMP] upgrade wkhtmltopdf to 0.12.5 for flectra 1.7.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ADD] stock_mrp_kit_product_short_availability gitlab CI/CD optimizations (psql12 / psql13) gitlab CI/CD optimizations gitlab CI/CD optimizations Stempeluhr PWA (WIP) MD5-Verschlüsselung des PIN in Python und Verprobung im Offline-Fall ebenfalls mit MD5-Vergleich Methoden _check_pin und _rpc_check_in_out offline verfügbar gemacht. Logout funktioniert aber Änderung wird noch nicht in syncCache DB gespeichert [IMP] use wkhtmltopdf 0.12.4 as 0.12.5 has memory errors Dockerfile for ubuntu and debian base images created and started with ubuntu_flectra Dockerfile, but not yet finished) re-enable ci stages [FIX] wkhtmltopdf error of ubuntu:bionic and also install libmagickwand-dev used by wand [REV] disable packaging job [IMP] use wkhtmltopdf 0.12.4 as 0.12.5 has memory errors Dockerfile for ubuntu and debian base images created and started with ubuntu_flectra Dockerfile, but not yet finished) re-enable ci stages [FIX] wkhtmltopdf error of ubuntu:bionic and also install libmagickwand-dev used by wand [REV] disable packaging job --- .gitlab-ci.yml | 311 ++++++++++++++++-- setup.py | 2 +- setup/docker/QUESTIONS.md | 4 + setup/docker/README.md | 94 ++++++ setup/docker/base/debian/Dockerfile | 62 ++++ setup/docker/base/ubuntu/Dockerfile | 50 +++ setup/docker/entrypoint.sh | 42 +++ setup/docker/flectra.conf | 37 +++ setup/docker/flectra/dev/Dockerfile | 35 ++ setup/docker/flectra/dev/bin/run.sh | 22 ++ setup/docker/flectra/dev/bin/set_root_pw.sh | 25 ++ setup/docker/flectra/ubuntu/Dockerfile | 56 ++++ setup/docker/gitlab-ci/gitlab_after_sucesss | 20 ++ setup/docker/gitlab-ci/gitlab_test_flectra.py | 307 +++++++++++++++++ setup/docker/requirements.txt | 49 +++ setup/package.py | 252 ++++++++++++++ 16 files changed, 1337 insertions(+), 31 deletions(-) create mode 100644 setup/docker/QUESTIONS.md create mode 100644 setup/docker/README.md create mode 100644 setup/docker/base/debian/Dockerfile create mode 100644 setup/docker/base/ubuntu/Dockerfile create mode 100755 setup/docker/entrypoint.sh create mode 100644 setup/docker/flectra.conf create mode 100644 setup/docker/flectra/dev/Dockerfile create mode 100644 setup/docker/flectra/dev/bin/run.sh create mode 100644 setup/docker/flectra/dev/bin/set_root_pw.sh create mode 100644 setup/docker/flectra/ubuntu/Dockerfile create mode 100755 setup/docker/gitlab-ci/gitlab_after_sucesss create mode 100755 setup/docker/gitlab-ci/gitlab_test_flectra.py create mode 100644 setup/docker/requirements.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5685fc15e9..918ee2cc8d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,35 +1,286 @@ -image: ubuntu:xenial stages: - - test -job_test_base_100: - stage: test + - unittest + - build + - integrationtest + - deploy + - package + +variables: + GITLAB_REGISTRY: "registry.gitlab.com/${CI_PROJECT_PATH}" + UBUNTU_BASE_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/base/ubuntu" + UBUNTU_FLECTRA_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/ubuntu" + UBUNTUDEV_FLECTRA_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/ubuntudev" + # DOCKER_HOST: tcp://docker:2375 + # DOCKER_DRIVER: overlay2 + + # Configure postgres service (https://hub.docker.com/_/postgres/) + POSTGRES_DB: postgres + POSTGRES_USER: flectra + POSTGRES_PASSWORD: flectra + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + +#-------------------------------------------------------------------- +# Job Templates +#-------------------------------------------------------------------- +.psql9:template: + services: &psql9_definition + - name: postgres:9-alpine + alias: postgres9 + +.psql10:template: + services: &psql10_definition + - name: postgres:10-alpine + alias: postgres10 + +.psql11:template: + services: &psql11_definition + - name: postgres:11-alpine + alias: postgres11 + +.psql12:template: + services: &psql12_definition + - name: postgres:12-alpine + alias: postgres12 + +.psql13:template: + services: &psql13_definition + - name: postgres:13-alpine + alias: postgres13 + +.unittest:template: &unittest_definition + image: ${UBUNTU_BASE_IMAGE}:latest + stage: unittest + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" + paths: + - flectra.log + when: always + expire_in: 2 week + coverage: '/^TOTAL.*\s+(\d+\%)$/' + only: + - branches + except: + - schedules + +.build:template: &build_definition + services: + - name: docker:dind + except: + - schedules + image: docker:stable + +.integrationtest:template: &integrationtest_definition + stage: integrationtest + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" + paths: + - flectra.log + when: always + expire_in: 2 week + only: + - tags + +.build:base:template: &build_base_definition + stage: build + services: + - name: docker:dind + only: + - schedules + image: docker:stable + +#-------------------------------------------------------------------- +# Jobs of stage unittest +#-------------------------------------------------------------------- +test:pg9_base: + <<: *unittest_definition + services: *psql9_definition script: - - apt-get update && apt-get install -y apt-utils locales git - - export POSTGRESQL_VERSION=10 - - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} - - gitlab_ci_install_flectra - - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=base" -job_test_all_100: - stage: test + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres9" + +test:pg9_all: + <<: *unittest_definition + services: *psql9_definition script: - - apt-get update && apt-get install -y apt-utils locales git - - export POSTGRESQL_VERSION=10 - - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} - - gitlab_ci_install_flectra - - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=all" -job_test_base_110: - stage: test + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres9" + +test:pg10_base: + <<: *unittest_definition + services: *psql10_definition script: - - apt-get update && apt-get install -y apt-utils locales git - - export POSTGRESQL_VERSION=11 - - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} - - gitlab_ci_install_flectra - - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=base" -job_test_all_110: - stage: test + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres10" + +test:pg10_all: + <<: *unittest_definition + services: *psql10_definition script: - - apt-get update && apt-get install -y apt-utils locales git - - export POSTGRESQL_VERSION=11 - - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} - - gitlab_ci_install_flectra - - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=all" + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres10" + +test:pg11_base: + <<: *unittest_definition + services: *psql11_definition + script: + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres11" + +test:pg11_all: + <<: *unittest_definition + services: *psql11_definition + script: + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres11" + +test:pg12_base: + <<: *unittest_definition + services: *psql12_definition + script: + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres12" + +test:pg12_all: + <<: *unittest_definition + services: *psql12_definition + script: + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres12" + +test:pg13_base: + <<: *unittest_definition + services: *psql13_definition + script: + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres13" + +test:pg13_all: + <<: *unittest_definition + services: *psql13_definition + script: + - cp -rp requirements.txt setup/docker/requirements.txt + - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres13" + +#-------------------------------------------------------------------- +# Jobs of stage build +#-------------------------------------------------------------------- +build:flectra_ubuntu: + <<: *build_definition + stage: build + only: + - tags + script: + - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker pull ${UBUNTU_FLECTRA_IMAGE}:latest || true + - docker build --cache-from ${UBUNTU_FLECTRA_IMAGE}:latest + --build-arg BASE_IMAGE=${UBUNTU_BASE_IMAGE} + --build-arg GITLAB_OWNER=${CI_PROJECT_NAMESPACE} + --build-arg GITLAB_COMMIT=${CI_COMMIT_SHA} + --file ./setup/docker/flectra/ubuntu/Dockerfile + --tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + ./setup/docker/ + - docker push ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + +#-------------------------------------------------------------------- +# Jobs of stage integrationtest +#-------------------------------------------------------------------- +itest:ubuntu_pg9_all: + <<: *integrationtest_definition + services: *psql9_definition + image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + script: + - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres9 + +itest:ubuntu_pg10_all: + <<: *integrationtest_definition + services: *psql10_definition + image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + script: + - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres10 + +itest:ubuntu_pg11_all: + <<: *integrationtest_definition + services: *psql11_definition + image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + script: + - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres11 + +itest:ubuntu_pg12_all: + <<: *integrationtest_definition + services: *psql12_definition + image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + script: + - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres12 + +#-------------------------------------------------------------------- +# Jobs of stage deploy +#-------------------------------------------------------------------- +deploy:flectra_gitlab: + <<: *build_definition + stage: deploy + needs: + - itest:ubuntu_pg10_all + only: + - tags + script: + - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_TAG + - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTU_FLECTRA_IMAGE}:latest + - docker push ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_TAG + - docker push ${UBUNTU_FLECTRA_IMAGE}:latest + +deploy:flectra_dockerhub: + <<: *build_definition + stage: deploy + needs: + - itest:ubuntu_pg10_all + only: + - tags + script: + - echo -n $DOCKER_HUB_PASSWORD | docker login -u $DOCKER_HUB_USER --password-stdin https://index.docker.io/v1/ + - docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA + - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${DOCKER_HUB_ROOT}/flectra:$CI_COMMIT_TAG + - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${DOCKER_HUB_ROOT}/flectra:latest + - docker push ${DOCKER_HUB_ROOT}/flectra:$CI_COMMIT_TAG + - docker push ${DOCKER_HUB_ROOT}/flectra:latest + +.deploy:flectra_bin: + services: + - name: docker:dind + - name: postgres:10-alpine + alias: postgres + only: + - tags + image: docker:stable + stage: package + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" + paths: + - build.log + when: always + expire_in: 1 day + script: + - apk add python3 python3-dev py-pip build-base rsync dpkg-dev rpm + - pip3 install pexpect + - mkdir /opt/flectra/ + - mkdir /opt/flectra/log + - mkdir /opt/flectra/build + - mkdir /opt/flectra/public + - cd ${CI_PROJECT_DIR}/setup && python3 ./package.py -b /opt/flectra/build -p /opt/flectra/public --no-windows > ${CI_PROJECT_DIR}/build.log + - find /opt/flectra/public -maxdepth 3 + + +#-------------------------------------------------------------------- +# Scheduled jobs of stage build (executed by gitlab scheduler) +#-------------------------------------------------------------------- + +build:base_ubuntu: + <<: *build_base_definition + script: + - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker pull ${UBUNTU_BASE_IMAGE}:latest || true + - docker build --cache-from ${UBUNTU_BASE_IMAGE}:latest -f ./setup/docker/base/ubuntu/Dockerfile ./setup/docker/ --tag ${UBUNTU_BASE_IMAGE}:latest + - docker push ${UBUNTU_BASE_IMAGE}:latest + diff --git a/setup.py b/setup.py index 13e3fea156..fdfeb09628 100644 --- a/setup.py +++ b/setup.py @@ -148,7 +148,7 @@ setup( 'passlib', 'pillow', # windows binary http://www.lfd.uci.edu/~gohlke/pythonlibs/ 'psutil', # windows binary code.google.com/p/psutil/downloads/list - 'psycopg2 >= 2.2', + 'psycopg2-binary >= 2.2', 'pydot', 'pyldap', # optional 'pyparsing', diff --git a/setup/docker/QUESTIONS.md b/setup/docker/QUESTIONS.md new file mode 100644 index 0000000000..0bdd0ef0de --- /dev/null +++ b/setup/docker/QUESTIONS.md @@ -0,0 +1,4 @@ +Questions about docker integration +================================== + +- how to avoid duplicate requirements.txt (e.g. setup/docker/requirements.txt) \ No newline at end of file diff --git a/setup/docker/README.md b/setup/docker/README.md new file mode 100644 index 0000000000..6f9629d782 --- /dev/null +++ b/setup/docker/README.md @@ -0,0 +1,94 @@ +Continuous Integration and Docker Image Generation +================================================== + + +Intro +----- + +This directory contains all the tools needed for optimized continuous integration testing +and direct deployment of docker images for development, test and production use. + +On every push or merge into core flectra repository some gitlab-ci jobs will be executed +to test the changes and generate new docker images. The details are described in the +following chapters. + +Goal +---- + +All code changes should be tested automatically in an acceptable duration and the revision +should be available with a few steps to run it as a docker container for manual testing. + +Docker Image Inheritance +------------------------ + +There are some base docker images which are used for better performance +(and less waiting time for developers). Based on that images other images are generated. + +The following image are deployed in gitlab registry for flectra: + +| Image | Tag | Base Image | Trigger | Content +|--------------|-------------|--------------------|-----------|------------ +| base/ubuntu | latest | ubuntu:xenial | scheduler | all required linux and python package to run flectra (ubuntu based) +| base/debian | latest | debian:jessie | scheduler | all required linux and python package to run flectra (debian based) +| ubuntu | **_build_** | base/ubuntu:latest | branch | full flectra installation and can be used for runbot like testing (ubuntu based) +| debian | **_build_** | base/debian:latest | branch | full flectra installation and can be used for runbot like testing (debian based) +| ubuntu | **_tag_** | base/ubuntu:latest | tag | full flectra installation for use in production (ubuntu based) +| debian | **_tag_** | base/debian:latest | tag | full flectra installation for use in production (debian based) + +some additional images for testing and building are used as services: + +- **postgres:9-alpine**: latest PostgreSQL 9.x image for running tests +- **postgres:10-alpine**: latest PostgreSQL 10.x image for running tests +- **docker:dind**: Special image containing docker to build and push images + +Stages +------ + +The whole CI and docker process is split in four parts (called stages): + +1. unittest: the jobs of this stage will validate the changes +2. build: at this stage the docker images are generated +3. integrationtest: these jobs will validate the flectra installation of docker images generated in previous stage +4. deploy: these jobs will deploy the docker images with flectra for production use (tagged wit git tag and as latest) + +Only when all jobs of one stage are successful done, the next stage is started. + +Jobs +---- + +### Stage: unittest + +The following jobs are started on every push or merge of code: + +- **test:pg9_base**: Run test of base module against PostgreSQL 9.x database +- **test:pg9_all**: Run test of all modules against PostgreSQL 9.x database +- **test:pg10_base**: Run test of base module against PostgreSQL 10.x database +- **test:pg9_all**: Run test of all modules against PostgreSQL 10.x database + +### Stage: build + +The following jobs are only started by gitlab scheduler: + +-**build:base_ubuntu**: Build and push base ubuntu image containing all linux and python requirements +-**build:base_debian**: Build and push base debian image containing all linux and python requirements + +The following jobs are started on push for a branch: + +-**build:flectra_ubuntu**: Build and push image with installed flectra, tagged with commit id (ubuntu based) +-**build:flectra_debian**: Build and push image with installed flectra, tagged with commit id (debian based) + +### Stage: integrationtest + +The following jobs are started on push for a branch: + +- **itest:ubuntu_pg9**: Run test of all modules against PostgreSQL 9.x database with flectra included in docker image (ubuntu based) +- **itest:ubuntu_pg10**: Run test of all modules against PostgreSQL 10.x database with flectra included in docker image (ubuntu based) +- **itest:debian_pg9**: Run test of all modules against PostgreSQL 9.x database with flectra included in docker image (debian based) +- **itest:debian_pg10**: Run test of all modules against PostgreSQL 10.x database with flectra included in docker image (debian based) + +### Stage: deploy + +The following jobs are started on push for tag: + +- **deploy:flectra_ubuntu**: Generate docker image tagged with git tag (=flectra version) and as latest (ubuntu based) +- **deploy:flectra_debian**: Generate docker image tagged with git tag (=flectra version) and as latest (debian based) diff --git a/setup/docker/base/debian/Dockerfile b/setup/docker/base/debian/Dockerfile new file mode 100644 index 0000000000..d83c225275 --- /dev/null +++ b/setup/docker/base/debian/Dockerfile @@ -0,0 +1,62 @@ +FROM debian:jessie + +LABEL maintainer="FlectraHQ " +LABEL authors="FlectraHQ,Jamotion" + +ARG POSTGRESQL_VERSION=9.6 + +RUN apt-get update && \ + apt-get install wget -y + +# adding postgresql debian repo in order to access all available postgresql version +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" > /etc/apt/sources.list.d/pgdg.list + +RUN apt-get update && \ + apt-get install git gnupg2 python3 curl expect-dev node-less \ + python3-pillow python3-lxml python-ldap3 python3-dev python3-pip python3-setuptools \ + libpq5 libpq-dev postgresql-${POSTGRESQL_VERSION} postgresql-client-${POSTGRESQL_VERSION} \ + postgresql-contrib-${POSTGRESQL_VERSION} postgresql-server-dev-${POSTGRESQL_VERSION} \ + fontconfig libxfont1 xfonts-encodings xfonts-utils xfonts-base xfonts-75dpi \ + npm nodejs libldap2-dev libsasl2-dev libxml2-dev libxslt1-dev libjpeg-dev zlib1g-dev libfontconfig1 -y + +RUN locale-gen --purge en_US.UTF-8 && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:UTF-8"\n' > /etc/default/locale + +ADD requirements.txt /tmp/requirements.txt + +RUN pip3 install --upgrade pip +RUN pip3 install passlib +RUN pip3 install -r /tmp/requirements.txt +RUN npm install -g less less-plugin-clean-css -y && \ + ln -s /usr/bin/nodejs /usr/bin/node + +RUN curl -o wkhtmltox.tar.xz -SL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \ + && echo '3f923f425d345940089e44c1466f6408b9619562 wkhtmltox.tar.xz' | sha1sum -c - \ + && tar xvf wkhtmltox.tar.xz \ + && cp wkhtmltox/lib/* /usr/local/lib/ \ + && cp wkhtmltox/bin/* /usr/local/bin/ \ + && cp -r wkhtmltox/share/man/man1 /usr/local/share/man/ + + +# install CI tools +RUN pip3 install coverage coveralls codecov simplejson pyserial pyyaml unittest2 pyopenssl>=16.2.0 && \ + wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz && \ + gunzip GeoLiteCity.dat.gz && \ + mkdir -p /usr/share/GeoIP/ && \ + mv GeoLiteCity.dat /usr/share/GeoIP/ + +RUN addgroup --gid=5500 flectra && adduser --system --uid=5500 --gid=5500 --home=/opt/flectra --shell=/bin/bash flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER root \ No newline at end of file diff --git a/setup/docker/base/ubuntu/Dockerfile b/setup/docker/base/ubuntu/Dockerfile new file mode 100644 index 0000000000..5265e97071 --- /dev/null +++ b/setup/docker/base/ubuntu/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:bionic + +LABEL maintainer="FlectraHQ " +LABEL authors="FlectraHQ,Jamotion" + +ARG POSTGRESQL_VERSION=10 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install wget -y + +RUN apt-get update && \ + apt-get install git gnupg2 python3 curl expect-dev node-less \ + python3-pillow python3-lxml python-ldap3 python3-dev python3-pip python3-setuptools python3-passlib \ + postgresql-${POSTGRESQL_VERSION} postgresql-server-dev-${POSTGRESQL_VERSION} \ + fontconfig libxfont2 xfonts-encodings xfonts-utils xfonts-base xfonts-75dpi \ + npm nodejs libldap2-dev libsasl2-dev libxml2-dev libxslt1-dev libjpeg-dev zlib1g-dev libfontconfig1 libmagickwand-dev -y + +RUN locale-gen --purge en_US.UTF-8 && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:UTF-8"\n' > /etc/default/locale + +ADD requirements.txt /tmp/requirements.txt + +RUN pip3 install --upgrade pip +RUN pip3 install -r /tmp/requirements.txt +RUN npm install -g less less-plugin-clean-css -y + +RUN curl -SsLO https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb && \ + apt-get install -f ./wkhtmltox_0.12.5-1.bionic_amd64.deb + +# install CI tools +RUN pip3 install coverage coveralls codecov simplejson pyserial pyyaml unittest2 pyopenssl>=16.2.0 \ + validate_email zpl2 unidecode shapely-geojson simplejson shapely geojson unidecode \ + email_validator wand imageio moviepy + +RUN addgroup --gid=5500 flectra && adduser --system --uid=5500 --gid=5500 --home=/opt/flectra --shell=/bin/bash flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER root \ No newline at end of file diff --git a/setup/docker/entrypoint.sh b/setup/docker/entrypoint.sh new file mode 100755 index 0000000000..01f388d275 --- /dev/null +++ b/setup/docker/entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e + +# set the postgres database host, port, user and password according to the environment +# and pass them as arguments to the flectra process if not present in the config file +: ${HOST:=${DB_PORT_5432_TCP_ADDR:='db'}} +: ${PORT:=${DB_PORT_5432_TCP_PORT:=5432}} +: ${USER:=${DB_ENV_POSTGRES_USER:=${POSTGRES_USER:='flectra'}}} +: ${PASSWORD:=${DB_ENV_POSTGRES_PASSWORD:=${POSTGRES_PASSWORD:='flectra'}}} + +DB_ARGS=() +function check_config() { + param="$1" + value="$2" + DB_ARGS+=("--${param}") + DB_ARGS+=("${value}") +} +if [[ -z "${IGNORE_ENV}" ]]; then + check_config "db_host" "$HOST" + check_config "db_port" "$PORT" + check_config "db_user" "$USER" + check_config "db_password" "$PASSWORD" +fi + +case "$1" in + -- | flectra) + shift + if [[ "$1" == "scaffold" ]] ; then + exec /opt/flectra/flectra-bin "$@" + else + exec /opt/flectra/flectra-bin "$@" "${DB_ARGS[@]}" + fi + ;; + -*) + exec /opt/flectra/flectra-bin "$@" "${DB_ARGS[@]}" + ;; + *) + exec "$@" +esac + +exit 1 diff --git a/setup/docker/flectra.conf b/setup/docker/flectra.conf new file mode 100644 index 0000000000..b7b571fd77 --- /dev/null +++ b/setup/docker/flectra.conf @@ -0,0 +1,37 @@ +[options] +addons_path = /mnt/extra-addons +data_dir = /var/lib/flectra +; admin_passwd = admin +; csv_internal_sep = , +; db_maxconn = 64 +; db_name = False +; db_template = template1 +; dbfilter = .* +; debug_mode = False +; email_from = False +; limit_memory_hard = 2684354560 +; limit_memory_soft = 2147483648 +; limit_request = 8192 +; limit_time_cpu = 60 +; limit_time_real = 120 +; list_db = True +; log_db = False +; log_handler = [':INFO'] +; log_level = info +; logfile = None +; longpolling_port = 7072 +; max_cron_threads = 2 +; osv_memory_age_limit = 1.0 +; osv_memory_count_limit = False +; smtp_password = False +; smtp_port = 25 +; smtp_server = localhost +; smtp_ssl = False +; smtp_user = False +; workers = 0 +; xmlrpc = True +; xmlrpc_interface = +; xmlrpc_port = 7073 +; xmlrpcs = True +; xmlrpcs_interface = +; xmlrpcs_port = 8071 diff --git a/setup/docker/flectra/dev/Dockerfile b/setup/docker/flectra/dev/Dockerfile new file mode 100644 index 0000000000..556077b5d3 --- /dev/null +++ b/setup/docker/flectra/dev/Dockerfile @@ -0,0 +1,35 @@ +#Partially from Odoo's dockerfile +#github.com/odoo/docker + +ARG BASE_IMAGE +ARG BASE_TAG +FROM ${BASE_IMAGE}:${BASE_TAG} +LABEL maintainer="FlectraHQ " +LABEL authors="FlectraHQ,Jamotion" + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER root + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +# Install and configure OpenSSH Server (for development mode only) +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install openssh-server pwgen build-essential +RUN mkdir -p /var/run/sshd \ + && sed -i "s/UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config \ + && sed -i "s/UsePAM.*/UsePAM no/g" /etc/ssh/sshd_config \ + && sed -i "s/PermitRootLogin.*/PermitRootLogin yes/g" /etc/ssh/sshd_config + +# add special scripts for development +ADD flectra/dev/bin /usr/local/bin + +VOLUME ["/opt/flectra", "/etc/flectra", "/var/log/flectra"] + +# Expose Flectra services +EXPOSE 7072 7073 22 + +ENTRYPOINT ["/usr/local/bin/run.sh"] diff --git a/setup/docker/flectra/dev/bin/run.sh b/setup/docker/flectra/dev/bin/run.sh new file mode 100644 index 0000000000..1e4ff76be5 --- /dev/null +++ b/setup/docker/flectra/dev/bin/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +if [ "${AUTHORIZED_KEYS}" != "**None**" ]; then + echo "=> Found authorized keys" + mkdir -p /root/.ssh + chmod 700 /root/.ssh + touch /root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys + IFS=$'\n' + arr=$(echo ${AUTHORIZED_KEYS} | tr "," "\n") + for x in $arr + do + x=$(echo $x |sed -e 's/^ *//' -e 's/ *$//') + cat /root/.ssh/authorized_keys | grep "$x" >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "=> Adding public key to /root/.ssh/authorized_keys: $x" + echo "$x" >> /root/.ssh/authorized_keys + fi + done +fi + +/usr/local/bin/set_root_pw.sh +exec /usr/sbin/sshd -D \ No newline at end of file diff --git a/setup/docker/flectra/dev/bin/set_root_pw.sh b/setup/docker/flectra/dev/bin/set_root_pw.sh new file mode 100644 index 0000000000..c6eb1a20b7 --- /dev/null +++ b/setup/docker/flectra/dev/bin/set_root_pw.sh @@ -0,0 +1,25 @@ +#!/bin/bash +if [ -f /etc/flectra/ssh-pwd ]; then + echo "Root password already saved. Using this one" + PASS=$( Setting a ${_word} password to the root user" +fi + +echo "root:$PASS" | chpasswd +echo "flectra:$PASS" | chpasswd +echo "=> Done!" +echo $PASS > /etc/flectra/ssh-pwd +IP=$(hostname --ip-address) +echo "========================================================================" +echo " You can now connect to this Developer Container via SSH using:" +echo "" +echo " ssh -p root@${IP}" +echo " or" +echo " ssh -p flectra@${IP}" +echo "" +echo " and enter the password '$PASS' when prompted" +echo "" +echo "========================================================================" \ No newline at end of file diff --git a/setup/docker/flectra/ubuntu/Dockerfile b/setup/docker/flectra/ubuntu/Dockerfile new file mode 100644 index 0000000000..d8c820b233 --- /dev/null +++ b/setup/docker/flectra/ubuntu/Dockerfile @@ -0,0 +1,56 @@ +#Partially from Odoo's dockerfile +#github.com/odoo/docker + +ARG BASE_IMAGE +FROM ${BASE_IMAGE}:latest +LABEL maintainer="FlectraHQ " +LABEL authors="FlectraHQ,Jamotion" + +ARG GITLAB_OWNER=flectra +ARG GITLAB_COMMIT=master +ARG ARG_DB_HOST=postgres +ARG ARG_DB_PORT=5432 +ARG ARG_DB_USER=flectra +ARG ARG_DB_PASSWORD=flectra + +ENV BRANCH=$ARG_BRANCH +ENV HOST=$ARG_DB_HOST +ENV PORT=$ARG_DB_PORT +ENV USER=$ARG_DB_USER +ENV PASSWORD=$ARG_DB_PASSWORD + +USER root +RUN mkdir -p /var/log/flectra && \ + chown -R flectra:flectra /var/log/flectra/ + +RUN locale-gen --purge en_US.UTF-8 && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:en"\n' > /etc/default/locale + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +USER flectra + +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en + +RUN cd /opt/flectra && \ + wget -q -O flectra.tar.gz https://gitlab.com/${GITLAB_OWNER}/flectra/-/archive/${GITLAB_COMMIT}/flectra.tar.gz && \ + tar -zxvf flectra.tar.gz flectra-${GITLAB_COMMIT}/flectra-bin flectra-${GITLAB_COMMIT}/flectra flectra-${GITLAB_COMMIT}/addons && \ + cd flectra-${GITLAB_COMMIT} && \ + mv addons/* flectra/addons/ && \ + mv flectra ../. && \ + mv flectra-bin ../. && \ + cd /opt/flectra && \ + rm -rf flectra-${GITLAB_COMMIT} + +EXPOSE 7073 7072 + +ADD entrypoint.sh /opt/flectra/entrypoint.sh + +ENTRYPOINT ["/opt/flectra/entrypoint.sh"] +CMD ["flectra"] + diff --git a/setup/docker/gitlab-ci/gitlab_after_sucesss b/setup/docker/gitlab-ci/gitlab_after_sucesss new file mode 100755 index 0000000000..3a36aa2bba --- /dev/null +++ b/setup/docker/gitlab-ci/gitlab_after_sucesss @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +# Part of code taken from github.com/OCA/maintainer-quality-tools + +import os + +from coverage.cmdline import main as coverage_main +from coveralls import cli as coveralls_cli +from codecov import main as codecov_main + + +coverage_main(["report", "--show-missing"]) +try: + coveralls_cli.main(argv=None) +except: + pass +try: + codecov_main(argv=None) +except: + pass diff --git a/setup/docker/gitlab-ci/gitlab_test_flectra.py b/setup/docker/gitlab-ci/gitlab_test_flectra.py new file mode 100755 index 0000000000..316fe95183 --- /dev/null +++ b/setup/docker/gitlab-ci/gitlab_test_flectra.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 + +# Part of code taken from github.com/OCA/maintainer-quality-tools + +import re +import os +import shutil +import subprocess +import sys +import uuid +import argparse +from coverage.cmdline import main as coverage_main +import locale + +try: + basestring +except NameError: + basestring = str + +RED = "\033[1;31m" +GREEN = "\033[1;32m" +YELLOW = "\033[1;33m" +YELLOW_LIGHT = "\033[33m" +CLEAR = "\033[0;m" + +# modules that are either added as dependencies for other modules, so better to ignore them for the moment +modules_to_ignore = ["account_analytic_default", "account_bank_statement_import", "account_budget", + "account_cancel", "account_check_printing", "account_payment", "account_tax_python", + "account_test", "account_voucher", "analytic", "anonymization", "association", + "auth_crypt", "auth_ldap", "auth_oauth", "auth_signup", "barcodes", "base_address_city", + "base_address_extended", "base_automation", "base_gengo", "base_geolocalize", "base_iban", + "base_import", "base_import_module", "base_setup", "base_sparse_field", "base_vat", + "base_vat_autocomplete", "bus", "calendar_sms", "crm_livechat", + "crm_phone_validation", "crm_project", "decimal_precision", "delivery", "document", "event", + "event_sale", "fetchmail", "gamification", "gamification_sale_crm", "google_account", + "google_calendar", "google_drive", "google_spreadsheet", + "hr_contract", "hr_expense_check", "hr_gamification", "hr_maintenance", + "hr_org_chart", "hr_payroll", "hr_payroll_account", "hr_recruitment_survey", + "hr_timesheet_attendance", "http_routing", "hw_blackbox_be", "hw_escpos", "hw_posbox_homepage", + "hw_posbox_upgrade", "hw_proxy", "hw_scale", "hw_scanner", "hw_screen", "link_tracker", + "live_currency_rate", "mass_mailing_event", "mass_mailing_event_track", "membership", + "mrp_byproduct", "note_pad", "pad", "pad_project", "password_security", + "payment", "payment_adyen", "payment_authorize", "payment_buckaroo", "payment_ogone", + "payment_paypal", "payment_payumoney", "payment_sips", "payment_stripe", "payment_transfer", + "phone_validation", "portal", "pos_cache", "pos_data_drinks", "pos_discount", "pos_mercury", + "pos_reprint", "pos_restaurant", "pos_sale", "procurement_jit", "product", "product_email_template", + "product_expiry", "product_extended", "product_margin", "project_timesheet_holidays", "rating", + "rating_project", "report_intrastat", "resource", "sale_mrp", "sale_order_dates", + "sale_payment", "sale_service_rating", "sale_timesheet", "sales_team", "sms", "stock_dropshipping", + "stock_landed_costs", "stock_picking_batch", "survey_crm", "theme_bootswatch", "theme_default", + "transifex", "utm", "web", "web_diagram", "web_editor", "web_kanban_gauge", "web_planner", + "web_settings_dashboard", "web_tour", "website_crm", "website_crm_partner_assign", + "website_crm_phone_validation", "website_customer", "website_event_questions", + "website_event_sale", "website_event_track", "website_form", "website_form_project", + "website_forum_doc", "website_gengo" , "website_google_map", "website_hr", "website_hr_recruitment", + "website_links", "website_livechat", "website_mail", "website_mail_channel", "website_mass_mailing", + "website_membership", "website_partner", "website_payment", "website_quote", "website_rating", + "website_rating_project", "website_sale_comparison", "website_sale_delivery", "website_sale_digital", + "website_sale_management", "website_sale_options", "website_sale_stock", "website_sale_stock_options", + "website_sale_wishlist", "website_theme_install", "website_twitter", "theme_art", "theme_hermit", + "theme_leith", "theme_techreceptives"] + + +def colorized(text, color): + return '\n'.join( + map(lambda line: color + line + CLEAR, text.split('\n'))) + + +def green(text): + return colorized(text, GREEN) + + +def yellow(text): + return colorized(text, YELLOW) + + +def red(text): + return colorized(text, RED) + + +def yellow_light(text): + return colorized(text, YELLOW_LIGHT) + + +fail_msg = red("FAIL") +success_msg = green("Success") + + +def has_test_errors(fname, dbname, check_loaded=True): + """ + Check a list of log lines for test errors. + Extension point to detect false positives. + """ + # Rules defining checks to perform + # this can be + # - a string which will be checked in a simple substring match + # - a regex object that will be matched against the whole message + # - a callable that receives a dictionary of the form + # { + # 'loglevel': ..., + # 'message': ...., + # } + errors_ignore = [ + 'Mail delivery failed', + 'failed sending mail', + ] + errors_report = [ + lambda x: 'loglevel' in x and x['loglevel'] == 'CRITICAL', + 'At least one test failed', + 'no access rules, consider adding one', + 'invalid module names, ignored', + ] + + def make_pattern_list_callable(pattern_list): + for i in range(len(pattern_list)): + if isinstance(pattern_list[i], basestring): + regex = re.compile(pattern_list[i]) + pattern_list[i] = lambda x, regex=regex: \ + regex.search(x['message']) + elif hasattr(pattern_list[i], 'match'): + regex = pattern_list[i] + pattern_list[i] = lambda x, regex=regex: \ + regex.search(x['message']) + + make_pattern_list_callable(errors_ignore) + make_pattern_list_callable(errors_report) + + print("-" * 10) + # Read log file removing ASCII color escapes: + # http://serverfault.com/questions/71285 + color_regex = re.compile(r'\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]') + log_start_regex = re.compile(r'^.*(?P(INFO|WARNING|DEBUG|ERROR|CRITICAL)).*?: (?P.*\S)\s*$') + log_records = [] + last_log_record = dict.fromkeys(log_start_regex.groupindex.keys()) + with open(fname, encoding='utf-8') as log: + for line in log: + line = color_regex.sub('', line) + if sys.stdout.encoding != 'UTF-8': + line = line.encode('ascii', 'ignore').decode('ascii') + match = log_start_regex.match(line) + if match: + last_log_record = match.groupdict() + log_records.append(last_log_record) + else: + last_log_record['message'] = '%s\n%s' % ( + last_log_record['message'], line.rstrip('\n') + ) + errors = [] + for log_record in log_records: + ignore = False + for ignore_pattern in errors_ignore: + if ignore_pattern(log_record): + ignore = True + break + if ignore: + break + print('{0}: {1}'.format(log_record['loglevel'], log_record['message'])) + for report_pattern in errors_report: + if report_pattern(log_record): + errors.append(log_record) + break + + if check_loaded: + if not [r for r in log_records if 'Modules loaded.' in r['message']]: + errors.append({'message': "Message not found: 'Modules loaded.'"}) + + if errors: + for e in errors: + print(e['message']) + print("-"*10) + return len(errors) + + +def run_flectra(type_of_db, server_path, host, port, user, password): + print("\n creating database for : ", type_of_db) + try: + db = str(uuid.uuid1()) + "-" + type_of_db + + # Setting locale is needed because of gitlab runner server have + # other locales + os.environ["LANG"] = "en_US.UTF-8" + os.environ["LANGUAGE"] = "en_US:en" + os.environ["LC_ALL"] = "en_US.UTF-8" + + os.environ["PGPASSWORD"] = password + subprocess.check_call( + ["createdb", "-U", user, "-h", host, "-p", port, "-T", + "template1", db]) + except subprocess.CalledProcessError: + print("Problem in creating database.") + else: + log_file = os.path.join(os.getcwd(), "flectra.log") + + # ugly hack as -i all is broken and does not take all modules into consideration + # need to remove l10n_* modules from the list as installing them all together + # creates trouble and build always fails + addons_path = os.path.join(server_path, "addons") + if not os.path.isdir(addons_path): + # Assume we are at integration test level where we can not test performance and client modules + addons_path = os.path.join(server_path, "flectra", "addons") + modules_to_ignore.append('test_performance') + modules_to_ignore.append('website') + + addons = list( + set([addon.name for addon in os.scandir(addons_path) if addon.is_dir() and "10n" not in addon.name]) - set( + modules_to_ignore)) + + if type_of_db == "all": + modules_to_init = str(",".join(addons)) + else: + modules_to_init = "base" + + cmd_flectra = ["unbuffer", "coverage", "run", "--source", server_path] + cmd_flectra += ["%s/flectra-bin" % (server_path), + "--db_host", + host, + "--db_port", + port, + "--db_user", + user, + "--db_password", + password, + "-d", + db, + '--db-filter', + db, + "--addons-path", + addons_path, + "--log-level=info", + "--stop-after-init", + "--test-enable", + "--init", modules_to_init] + print("CMD EXECUTED --->>> ", " ".join(cmd_flectra)) + + with open(log_file, 'w') as outfile: + with subprocess.Popen( + cmd_flectra, + stdout=outfile, + bufsize=1, + universal_newlines=True) as p: + returncode = p.wait() + outfile.close() + errors = has_test_errors(os.path.join(server_path, log_file), db) + return {'errors': errors, 'returncode': returncode} + + +def main(argv=None): + parser = argparse.ArgumentParser(description='Create Docker Instances....') + parser.add_argument('--build', dest='build', help='Port For Instance', required=True, default="base") + parser.add_argument('--server-path', dest='path', help='Flectra Path') + parser.add_argument('--host', dest='host', + help='PostgreSQL Host', + default="postgres") + parser.add_argument('--port', dest='port', + help='PostgreSQL Port', + default="5432") + parser.add_argument('--user', dest='user', + help='PostgreSQL user', + default="flectra") + parser.add_argument('--password', dest='password', + help='PostgreSQL password', + default="flectra") + args = parser.parse_args() + build = args.build + + if argv is None: + argv = sys.argv + res = {} + server_path = args.path or os.getcwd() + + if build == "all": + res['all'] = run_flectra( + type_of_db="all", + server_path=server_path, + host=args.host, + port=args.port, + user=args.user, + password=args.password, + ) + errors = 'all' in res and res['all'] and 'errors' in res['all'] and \ + res['all']['errors'] + + if build == "base": + res['base'] = run_flectra( + type_of_db="base", + server_path=server_path, + host=args.host, + port=args.port, + user=args.user, + password=args.password, + ) + errors = 'base' in res and res['base'] and 'errors' in res['base'] and \ + res['base']['errors'] + + print("************ ERRORS ************ : ", errors) + return errors + + +if __name__ == '__main__': + rc = main() + + if rc: + exit(rc) + + coverage_main(["report"]) + + exit(rc) diff --git a/setup/docker/requirements.txt b/setup/docker/requirements.txt new file mode 100644 index 0000000000..307eb7ba78 --- /dev/null +++ b/setup/docker/requirements.txt @@ -0,0 +1,49 @@ +Babel==2.3.4 +decorator==4.0.10 +docutils==0.12 +ebaysdk==2.1.5 +feedparser==5.2.1 +gevent==1.1.2 ; sys_platform != 'win32' and python_version < '3.7' +gevent==1.3.4 ; sys_platform != 'win32' and python_version >= '3.7' +greenlet==0.4.10 ; python_version < '3.7' +greenlet==0.4.13 ; python_version >= '3.7' +html2text==2016.9.19 +Jinja2==2.8.1 +lxml==3.7.1 ; sys_platform != 'win32' and python_version < '3.7' +lxml==4.2.3 ; sys_platform != 'win32' and python_version >= '3.7' +lxml ; sys_platform == 'win32' +Mako==1.0.4 +MarkupSafe==0.23 +mock==2.0.0 +num2words==0.5.6 +oauthlib==2.0.7 +ofxparse==0.16 +passlib==1.6.5 +phonenumbers==8.8.9 +Pillow==4.0.0 +psutil==4.3.1; sys_platform != 'win32' +psycopg2==2.7.3.1; sys_platform != 'win32' +pydot==1.2.3 +pyldap==2.4.28; sys_platform != 'win32' +pyparsing==2.1.10 +PyPDF2==1.26.0 +pyserial==3.1.1 +python-dateutil==2.5.3 +pytz==2016.7 +pyusb==1.0.0 +PyYAML==3.12 ; python_version < '3.7' +PyYAML==3.13 ; python_version >= '3.7' +qrcode==5.3 +reportlab==3.3.0 +requests==2.20.0 +suds-jurko==0.6 +vatnumber==1.2 +vobject==0.9.3 +Werkzeug==0.11.15 +XlsxWriter==0.9.3 +xlwt==1.3.* +xlrd==1.0.0 +unittest2==1.1.0 +numpy==1.14.3 +pypiwin32 ; sys_platform == 'win32' +pycrypto==2.6.1 \ No newline at end of file diff --git a/setup/package.py b/setup/package.py index 37999a0151..019661b1ef 100644 --- a/setup/package.py +++ b/setup/package.py @@ -101,6 +101,258 @@ def publish(args, pub_type, extensions): published.append(_publish(release[0])) return published +class FlectraDocker(object): + def __init__(self): + self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False) + self.port = 7073 # TODO sle: reliable way to get a free port? + self.prompt_re = '[root@nightly-tests] # ' + self.timeout = 600 + + def system(self, command): + self.docker.sendline(command) + self.docker.expect_exact(self.prompt_re) + + def start(self, docker_image, build_dir, pub_dir): + self.build_dir = build_dir + self.pub_dir = pub_dir + + self.docker = pexpect.spawn( + 'docker run -v %s:/opt/release -p 127.0.0.1:%s:7073' + ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image), + timeout=self.timeout, + searchwindowsize=len(self.prompt_re) + 1, + ) + time.sleep(2) # let the bash start + self.docker.logfile_read = self.log_file + self.id = subprocess.check_output('docker ps -l -q', shell=True).strip().decode('ascii') + + def end(self): + try: + _rpc_count_modules(port=str(self.port)) + except Exception as e: + logging.error('Exception during docker execution: %s:' % str(e)) + logging.error('Error during docker execution: printing the bash output:') + with open(self.log_file.name) as f: + print('\n'.join(f.readlines()), file=stderr) + raise + finally: + self.docker.close() + system('docker rm -f %s' % self.id) + self.log_file.close() + os.remove(self.log_file.name) + +@contextmanager +def docker(docker_image, build_dir, pub_dir): + _docker = FlectraDocker() + try: + _docker.start(docker_image, build_dir, pub_dir) + try: + yield _docker + except Exception: + raise + finally: + _docker.end() + +class KVM(object): + def __init__(self, o, image, ssh_key='', login='flectra'): + self.o = o + self.image = image + self.ssh_key = ssh_key + self.login = login + + def timeout(self,signum,frame): + logging.warning("vm timeout kill",self.pid) + os.kill(self.pid,15) + + def start(self): + l="kvm -cpu core2duo -smp 2,sockets=2,cores=1,threads=1 -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:17073-:7073,hostfwd=tcp:127.0.0.1:15432-:5432 -m 1024 -drive".split(" ") + #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image) + l.append('file=%s,snapshot=on'%self.image) + #l.extend(['-vnc','127.0.0.1:1']) + l.append('-nographic') + logging.info("Starting kvm: {}".format( " ".join(l))) + self.pid=os.spawnvp(os.P_NOWAIT, l[0], l) + time.sleep(50) + signal.alarm(2400) + signal.signal(signal.SIGALRM, self.timeout) + try: + self.run() + finally: + signal.signal(signal.SIGALRM, signal.SIG_DFL) + os.kill(self.pid,15) + time.sleep(10) + + def ssh(self,cmd): + l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd] + system(l) + + def rsync(self,args,options='--delete --exclude .git --exclude .tx --exclude __pycache__'): + cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args) + system(cmd) + + def run(self): + pass + +class KVMWinBuildExe(KVM): + def run(self): + with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f: + f.write("VERSION=%s\n" % version.replace('~', '_').replace('+', '')) + with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f: + f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', '')) + with open(join(self.o.build_dir, 'setup/win32/Makefile.servicename'), 'w') as f: + f.write("SERVICENAME=%s\n" % nt_service_name) + + remote_build_dir = '/cygdrive/c/flectrabuild/server/' + + self.ssh("mkdir -p build") + logging.info("Syncing Flectra files to virtual machine...") + self.rsync('%s/ %s@127.0.0.1:%s' % (self.o.build_dir, self.login, remote_build_dir)) + self.ssh("cd {}setup/win32;time make allinone;".format(remote_build_dir)) + self.rsync('%s@127.0.0.1:%ssetup/win32/release/ %s/' % (self.login, remote_build_dir, self.o.build_dir), '') + logging.info("KVMWinBuildExe.run(): done") + +class KVMWinTestExe(KVM): + def run(self): + setuppath = glob("%s/flectra-server-setup-*.exe" % self.o.build_dir)[0] + setupfile = setuppath.split('/')[-1] + setupversion = setupfile.split('flectra-server-setup-')[1].split('.exe')[0] + + self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login)) + self.ssh("TEMP=/tmp ./%s /S" % setupfile) + self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Flectra %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion) + self.ssh('netsh advfirewall set publicprofile state off') + self.ssh('/cygdrive/c/"Program Files"/"Flectra {sv}"/python/python.exe \'c:\\Program Files\\Flectra {sv}\\server\\flectra-bin\' -d mycompany -i base --stop-after-init'.format(sv=setupversion)) + _rpc_count_modules(port=17073) + +#---------------------------------------------------------- +# Stage: building +#---------------------------------------------------------- +def _prepare_build_dir(o, win32=False): + + cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo'] + if not win32: + cmd += ['--exclude', 'setup/win32'] + system(cmd + ['%s/' % o.flectra_dir, o.build_dir]) + for addon_path in glob(join(o.build_dir, 'addons/*')): + if addon_path.split(os.path.sep)[-1] not in ADDONS_NOT_TO_PUBLISH: + try: + shutil.move(addon_path, join(o.build_dir, 'flectra/addons')) + except shutil.Error as e: + # Thrown when the add-on is already in flectra/addons (if _prepare_build_dir + # has already been called once) + logging.warning("Warning '{}' while moving addon '{}'".format(e,addon_path)) + if addon_path.startswith(o.build_dir) and os.path.isdir(addon_path): + logging.info("Removing '{}'".format(addon_path)) + try: + shutil.rmtree(addon_path) + except shutil.Error as rm_error: + logging.warning("Cannot remove '{}': {}".format(addon_path, rm_error)) + +def build_tgz(o): + system(['python3', 'setup.py', 'sdist', '--quiet', '--formats=gztar,zip'], o.build_dir) + system(['mv', glob('%s/dist/flectra-*.tar.gz' % o.build_dir)[0], '%s/flectra_%s.%s.tar.gz' % (o.build_dir, version, timestamp)]) + system(['mv', glob('%s/dist/flectra-*.zip' % o.build_dir)[0], '%s/flectra_%s.%s.zip' % (o.build_dir, version, timestamp)]) + +def build_deb(o): + # Append timestamp to version for the .dsc to refer the right .tar.gz + cmd=['sed', '-i', '1s/^.*$/flectra (%s.%s) stable; urgency=low/'%(version,timestamp), 'debian/changelog'] + subprocess.call(cmd, cwd=o.build_dir) + if not o.no_debsign: + deb = pexpect.spawn('dpkg-buildpackage -rfakeroot -k%s' % GPGID, cwd=o.build_dir) + deb.logfile = stdout.buffer + if GPGPASSPHRASE: + deb.expect_exact('Passphrase: ', timeout=1200) + deb.send(GPGPASSPHRASE + '\r\n') + deb.expect(pexpect.EOF, timeout=1200) + else: + subprocess.call(['dpkg-buildpackage', '-rfakeroot', '-uc', '-us'], cwd=o.build_dir) + # As the packages are built in the parent of the buildir, we move them back to build_dir + build_dir_parent = '{}/../'.format(o.build_dir) + wildcards = ['flectra_{}'.format(wc) for wc in ('*.deb', '*.dsc', '*_amd64.changes', '*.tar.gz', '*.tar.xz')] + move_glob(build_dir_parent, wildcards, o.build_dir) + +def build_rpm(o): + system(['python3', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir) + system(['mv', glob('%s/dist/flectra-*.noarch.rpm' % o.build_dir)[0], '%s/flectra_%s.%s.noarch.rpm' % (o.build_dir, version, timestamp)]) + +def build_exe(o): + KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start() + system(['cp', glob('%s/flectra*.exe' % o.build_dir)[0], '%s/flectra_%s.%s.exe' % (o.build_dir, version, timestamp)]) + +#---------------------------------------------------------- +# Stage: testing +#---------------------------------------------------------- +def _prepare_testing(o): + logging.info('Preparing testing') + if not o.no_tarball: + logging.info('Preparing docker container instance for tarball') + subprocess.call(["mkdir", "docker_src"], cwd=o.build_dir) + subprocess.call(["cp", "package.dfsrc", os.path.join(o.build_dir, "docker_src", "Dockerfile")], + cwd=os.path.join(o.flectra_dir, "setup")) + # Use rsync to copy requirements.txt in order to keep original permissions + subprocess.call(["rsync", "-a", "requirements.txt", os.path.join(o.build_dir, "docker_src")], + cwd=os.path.join(o.flectra_dir)) + subprocess.call(["docker", "build", "-t", "flectra-%s-src-nightly-tests" % docker_version, "."], + cwd=os.path.join(o.build_dir, "docker_src")) + if not o.no_debian: + logging.info('Preparing docker container instance for debian') + subprocess.call(["mkdir", "docker_debian"], cwd=o.build_dir) + subprocess.call(["cp", "package.dfdebian", os.path.join(o.build_dir, "docker_debian", "Dockerfile")], + cwd=os.path.join(o.flectra_dir, "setup")) + # Use rsync to copy requirements.txt in order to keep original permissions + subprocess.call(["rsync", "-a", "requirements.txt", os.path.join(o.build_dir, "docker_debian")], + cwd=os.path.join(o.flectra_dir)) + subprocess.call(["docker", "build", "-t", "flectra-%s-debian-nightly-tests" % docker_version, "."], + cwd=os.path.join(o.build_dir, "docker_debian")) + if not o.no_rpm: + logging.info('Preparing docker container instance for RPM') + subprocess.call(["mkdir", "docker_fedora"], cwd=o.build_dir) + subprocess.call(["cp", "package.dffedora", os.path.join(o.build_dir, "docker_fedora", "Dockerfile")], + cwd=os.path.join(o.flectra_dir, "setup")) + subprocess.call(["docker", "build", "-t", "flectra-%s-fedora-nightly-tests" % docker_version, "."], + cwd=os.path.join(o.build_dir, "docker_fedora")) + +def test_tgz(o): + logging.info('Testing tarball in docker') + with docker('flectra-%s-src-nightly-tests' % docker_version, o.build_dir, o.pub) as wheezy: + wheezy.release = '*.tar.gz' + wheezy.system("service postgresql start") + wheezy.system('pip3 install /opt/release/%s' % wheezy.release) + wheezy.system("useradd --system --no-create-home flectra") + wheezy.system('su postgres -s /bin/bash -c "createuser -s flectra"') + wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"') + wheezy.system('mkdir /var/lib/flectra') + wheezy.system('chown flectra:flectra /var/lib/flectra') + wheezy.system('su flectra -s /bin/bash -c "flectra -d mycompany -i base --stop-after-init"') + wheezy.system('su flectra -s /bin/bash -c "flectra -d mycompany &"') + +def test_deb(o): + logging.info('Testing deb package in docker') + with docker('flectra-%s-debian-nightly-tests' % docker_version, o.build_dir, o.pub) as wheezy: + wheezy.release = '*.deb' + wheezy.system("service postgresql start") + wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"') + wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release) + wheezy.system('/usr/bin/apt-get install -f -y') + wheezy.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -i base --stop-after-init"') + wheezy.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany &"') + +def test_rpm(o): + logging.info('Testing rpm in docker') + with docker('flectra-%s-fedora-nightly-tests' % docker_version, o.build_dir, o.pub) as fedora24: + fedora24.release = '*.noarch.rpm' + # Start postgresql + fedora24.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"') + fedora24.system('sleep 5') + fedora24.system('su postgres -c "createdb mycompany"') + # Flectra install + fedora24.system('dnf install -d 0 -e 0 /opt/release/%s -y' % fedora24.release) + fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -i base --stop-after-init"') + fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany &"') + +def test_exe(o): + logging.info('Testng windows installer in KVM') + KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start() # --------------------------------------------------------- # Generates Packages, Sources and Release files of debian package