[IMP] upgrade wkhtmltopdf to 0.12.5 for flectra 1.7.x

[ADD] stock_mrp_kit_product_short_availability

gitlab CI/CD optimizations (psql12 / psql13)

gitlab CI/CD optimizations

gitlab CI/CD optimizations

Stempeluhr PWA (WIP)
MD5-Verschlüsselung des PIN in Python und Verprobung im Offline-Fall ebenfalls mit MD5-Vergleich
Methoden _check_pin und _rpc_check_in_out offline verfügbar gemacht.
Logout funktioniert aber Änderung wird noch nicht in syncCache DB gespeichert

[IMP] use wkhtmltopdf 0.12.4 as 0.12.5 has memory errors

Dockerfile for ubuntu and debian base images created and started with
ubuntu_flectra Dockerfile, but not yet finished)

re-enable ci stages

[FIX] wkhtmltopdf error of ubuntu:bionic
and also install libmagickwand-dev used by wand

[REV] disable packaging job

[IMP] use wkhtmltopdf 0.12.4 as 0.12.5 has memory errors

Dockerfile for ubuntu and debian base images created and started with
ubuntu_flectra Dockerfile, but not yet finished)

re-enable ci stages

[FIX] wkhtmltopdf error of ubuntu:bionic
and also install libmagickwand-dev used by wand

[REV] disable packaging job

.gitlab-ci.yml

@@ -1,35 +1,286 @@
-image: ubuntu:xenial
 stages:
-  - test
-job_test_base_100:
-  stage: test
+  - unittest
+  - build
+  - integrationtest
+  - deploy
+  - package
+
+variables:
+  GITLAB_REGISTRY: "registry.gitlab.com/${CI_PROJECT_PATH}"
+  UBUNTU_BASE_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/base/ubuntu"
+  UBUNTU_FLECTRA_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/ubuntu"
+  UBUNTUDEV_FLECTRA_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/ubuntudev"
+  #  DOCKER_HOST: tcp://docker:2375
+  #  DOCKER_DRIVER: overlay2
+
+  # Configure postgres service (https://hub.docker.com/_/postgres/)
+  POSTGRES_DB: postgres
+  POSTGRES_USER: flectra
+  POSTGRES_PASSWORD: flectra
+  POSTGRES_HOST: postgres
+  POSTGRES_PORT: "5432"
+
+#--------------------------------------------------------------------
+# Job Templates
+#--------------------------------------------------------------------
+.psql9:template:
+  services: &psql9_definition
+    - name: postgres:9-alpine
+      alias: postgres9
+
+.psql10:template:
+  services: &psql10_definition
+    - name: postgres:10-alpine
+      alias: postgres10
+
+.psql11:template:
+  services: &psql11_definition
+    - name: postgres:11-alpine
+      alias: postgres11
+
+.psql12:template:
+  services: &psql12_definition
+    - name: postgres:12-alpine
+      alias: postgres12
+
+.psql13:template:
+  services: &psql13_definition
+    - name: postgres:13-alpine
+      alias: postgres13
+
+.unittest:template: &unittest_definition
+  image: ${UBUNTU_BASE_IMAGE}:latest
+  stage: unittest
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    paths:
+      - flectra.log
+    when: always
+    expire_in: 2 week
+  coverage: '/^TOTAL.*\s+(\d+\%)$/'
+  only:
+    - branches
+  except:
+    - schedules
+
+.build:template: &build_definition
+  services:
+    - name: docker:dind
+  except:
+    - schedules
+  image: docker:stable
+
+.integrationtest:template: &integrationtest_definition
+  stage: integrationtest
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    paths:
+      - flectra.log
+    when: always
+    expire_in: 2 week
+  only:
+    - tags
+
+.build:base:template: &build_base_definition
+  stage: build
+  services:
+    - name: docker:dind
+  only:
+    - schedules
+  image: docker:stable
+
+#--------------------------------------------------------------------
+# Jobs of stage unittest
+#--------------------------------------------------------------------
+test:pg9_base:
+  <<: *unittest_definition
+  services: *psql9_definition
   script:
-  - apt-get update && apt-get install -y apt-utils locales git
-  - export POSTGRESQL_VERSION=10
-  - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH}
-  - gitlab_ci_install_flectra
-  - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=base"
-job_test_all_100:
-  stage: test
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres9"
+
+test:pg9_all:
+  <<: *unittest_definition
+  services: *psql9_definition
   script:
-  - apt-get update && apt-get install -y apt-utils locales git
-  - export POSTGRESQL_VERSION=10
-  - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH}
-  - gitlab_ci_install_flectra
-  - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=all"
-job_test_base_110:
-  stage: test
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres9"
+
+test:pg10_base:
+  <<: *unittest_definition
+  services: *psql10_definition
   script:
-  - apt-get update && apt-get install -y apt-utils locales git
-  - export POSTGRESQL_VERSION=11
-  - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH}
-  - gitlab_ci_install_flectra
-  - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=base"
-job_test_all_110:
-  stage: test
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres10"
+
+test:pg10_all:
+  <<: *unittest_definition
+  services: *psql10_definition
   script:
-  - apt-get update && apt-get install -y apt-utils locales git
-  - export POSTGRESQL_VERSION=11
-  - git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH}
-  - gitlab_ci_install_flectra
-  - su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=all"
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres10"
+
+test:pg11_base:
+  <<: *unittest_definition
+  services: *psql11_definition
+  script:
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres11"
+
+test:pg11_all:
+  <<: *unittest_definition
+  services: *psql11_definition
+  script:
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres11"
+
+test:pg12_base:
+  <<: *unittest_definition
+  services: *psql12_definition
+  script:
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres12"
+
+test:pg12_all:
+  <<: *unittest_definition
+  services: *psql12_definition
+  script:
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres12"
+
+test:pg13_base:
+  <<: *unittest_definition
+  services: *psql13_definition
+  script:
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres13"
+
+test:pg13_all:
+  <<: *unittest_definition
+  services: *psql13_definition
+  script:
+    - cp -rp requirements.txt setup/docker/requirements.txt
+    - su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres13"
+
+#--------------------------------------------------------------------
+# Jobs of stage build
+#--------------------------------------------------------------------
+build:flectra_ubuntu:
+  <<: *build_definition
+  stage: build
+  only:
+    - tags
+  script:
+    - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
+    - docker pull ${UBUNTU_FLECTRA_IMAGE}:latest || true
+    - docker build --cache-from ${UBUNTU_FLECTRA_IMAGE}:latest
+      --build-arg BASE_IMAGE=${UBUNTU_BASE_IMAGE}
+      --build-arg GITLAB_OWNER=${CI_PROJECT_NAMESPACE}
+      --build-arg GITLAB_COMMIT=${CI_COMMIT_SHA}
+      --file ./setup/docker/flectra/ubuntu/Dockerfile
+      --tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
+      ./setup/docker/
+    - docker push ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
+
+#--------------------------------------------------------------------
+# Jobs of stage integrationtest
+#--------------------------------------------------------------------
+itest:ubuntu_pg9_all:
+  <<: *integrationtest_definition
+  services: *psql9_definition
+  image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
+  script:
+    - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres9
+
+itest:ubuntu_pg10_all:
+  <<: *integrationtest_definition
+  services: *psql10_definition
+  image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
+  script:
+    - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres10
+
+itest:ubuntu_pg11_all:
+  <<: *integrationtest_definition
+  services: *psql11_definition
+  image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
+  script:
+    - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres11
+
+itest:ubuntu_pg12_all:
+  <<: *integrationtest_definition
+  services: *psql12_definition
+  image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
+  script:
+    - cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres12
+
+#--------------------------------------------------------------------
+# Jobs of stage deploy
+#--------------------------------------------------------------------
+deploy:flectra_gitlab:
+  <<: *build_definition
+  stage: deploy
+  needs:
+    - itest:ubuntu_pg10_all
+  only:
+    - tags
+  script:
+    - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
+    - docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
+    - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_TAG
+    - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTU_FLECTRA_IMAGE}:latest
+    - docker push ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_TAG
+    - docker push ${UBUNTU_FLECTRA_IMAGE}:latest
+
+deploy:flectra_dockerhub:
+  <<: *build_definition
+  stage: deploy
+  needs:
+    - itest:ubuntu_pg10_all
+  only:
+    - tags
+  script:
+    - echo -n $DOCKER_HUB_PASSWORD | docker login -u $DOCKER_HUB_USER --password-stdin https://index.docker.io/v1/
+    - docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
+    - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${DOCKER_HUB_ROOT}/flectra:$CI_COMMIT_TAG
+    - docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${DOCKER_HUB_ROOT}/flectra:latest
+    - docker push ${DOCKER_HUB_ROOT}/flectra:$CI_COMMIT_TAG
+    - docker push ${DOCKER_HUB_ROOT}/flectra:latest
+
+.deploy:flectra_bin:
+  services:
+    - name: docker:dind
+    - name: postgres:10-alpine
+      alias: postgres
+  only:
+    - tags
+  image: docker:stable
+  stage: package
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    paths:
+      - build.log
+    when: always
+    expire_in: 1 day
+  script:
+    - apk add python3 python3-dev py-pip build-base rsync dpkg-dev rpm
+    - pip3 install pexpect
+    - mkdir /opt/flectra/
+    - mkdir /opt/flectra/log
+    - mkdir /opt/flectra/build
+    - mkdir /opt/flectra/public
+    - cd ${CI_PROJECT_DIR}/setup && python3 ./package.py -b /opt/flectra/build -p /opt/flectra/public --no-windows > ${CI_PROJECT_DIR}/build.log
+    - find /opt/flectra/public -maxdepth 3
+
+
+#--------------------------------------------------------------------
+# Scheduled jobs of stage build (executed by gitlab scheduler)
+#--------------------------------------------------------------------
+
+build:base_ubuntu:
+  <<: *build_base_definition
+  script:
+    - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
+    - docker pull ${UBUNTU_BASE_IMAGE}:latest || true
+    - docker build --cache-from ${UBUNTU_BASE_IMAGE}:latest -f ./setup/docker/base/ubuntu/Dockerfile ./setup/docker/ --tag ${UBUNTU_BASE_IMAGE}:latest
+    - docker push ${UBUNTU_BASE_IMAGE}:latest
+

setup.py

@@ -148,7 +148,7 @@ setup(
         'passlib',
         'pillow',  # windows binary http://www.lfd.uci.edu/~gohlke/pythonlibs/
         'psutil',  # windows binary code.google.com/p/psutil/downloads/list
-        'psycopg2 >= 2.2',
+        'psycopg2-binary >= 2.2',
         'pydot',
         'pyldap',  # optional
         'pyparsing',

setup/docker/QUESTIONS.md

@@ -0,0 +1,4 @@
+Questions about docker integration
+==================================
+
+-  how to avoid duplicate requirements.txt (e.g. setup/docker/requirements.txt)

setup/docker/README.md

@@ -0,0 +1,94 @@
+Continuous Integration and Docker Image Generation
+==================================================
+
+
+Intro
+-----
+
+This directory contains all the tools needed for optimized continuous integration testing 
+and direct deployment of docker images for development, test and production use.
+
+On every push or merge into core flectra repository some gitlab-ci jobs will be executed
+to test the changes and generate new docker images. The details are described in the 
+following chapters.  
+
+Goal
+----
+
+All code changes should be tested automatically in an acceptable duration and the revision 
+should be available with a few steps to run it as a docker container for manual testing. 
+ 
+Docker Image Inheritance
+------------------------
+
+There are some base docker images which are used for better performance 
+(and less waiting time for developers). Based on that images other images are generated.
+
+The following image are deployed in gitlab registry for flectra: 
+
+| Image        | Tag         | Base Image         | Trigger   | Content 
+|--------------|-------------|--------------------|-----------|------------
+|  base/ubuntu | latest      | ubuntu:xenial      | scheduler | all required linux and python package to run flectra (ubuntu based) 
+|  base/debian | latest      | debian:jessie      | scheduler | all required linux and python package to run flectra (debian based)
+|  ubuntu      | **_build_** | base/ubuntu:latest | branch    | full flectra installation and can be used for runbot like testing (ubuntu based)
+|  debian      | **_build_** | base/debian:latest | branch    | full flectra installation and can be used for runbot like testing (debian based)
+|  ubuntu      | **_tag_**   | base/ubuntu:latest | tag       | full flectra installation for use in production (ubuntu based)
+|  debian      | **_tag_**   | base/debian:latest | tag       | full flectra installation for use in production (debian based)
+
+some additional images for testing and building are used as services:
+
+- **postgres:9-alpine**: latest PostgreSQL 9.x image for running tests  
+- **postgres:10-alpine**: latest PostgreSQL 10.x image for running tests
+- **docker:dind**: Special image containing docker to build and push images  
+ 
+Stages
+------
+
+The whole CI and docker process is split in four parts (called stages):
+
+1. unittest: the jobs of this stage will validate the changes 
+2. build: at this stage the docker images are generated
+3. integrationtest: these jobs will validate the flectra installation of docker images generated in previous stage
+4. deploy: these jobs will deploy the docker images with flectra for production use (tagged wit git tag and as latest)   
+
+Only when all jobs of one stage are successful done, the next stage is started.
+
+Jobs
+----
+
+### Stage: unittest
+
+The following jobs are started on every push or merge of code:
+
+- **test:pg9_base**: Run test of base module against PostgreSQL 9.x database
+- **test:pg9_all**: Run test of all modules against PostgreSQL 9.x database
+- **test:pg10_base**: Run test of base module against PostgreSQL 10.x database
+- **test:pg9_all**: Run test of all modules against PostgreSQL 10.x database
+
+### Stage: build
+
+The following jobs are only started by gitlab scheduler:
+
+-**build:base_ubuntu**: Build and push base ubuntu image containing all linux and python requirements
+-**build:base_debian**: Build and push base debian image containing all linux and python requirements
+
+The following jobs are started on push for a branch:
+
+-**build:flectra_ubuntu**: Build and push image with installed flectra, tagged with commit id (ubuntu based)
+-**build:flectra_debian**: Build and push image with installed flectra, tagged with commit id (debian based)
+
+### Stage: integrationtest
+
+The following jobs are started on push for a branch:
+
+- **itest:ubuntu_pg9**: Run test of all modules against PostgreSQL 9.x database with flectra included in docker image (ubuntu based)
+- **itest:ubuntu_pg10**: Run test of all modules against PostgreSQL 10.x database with flectra included in docker image (ubuntu based)
+- **itest:debian_pg9**: Run test of all modules against PostgreSQL 9.x database with flectra included in docker image (debian based)
+- **itest:debian_pg10**: Run test of all modules against PostgreSQL 10.x database with flectra included in docker image (debian based)
+
+### Stage: deploy
+
+The following jobs are started on push for tag:
+
+- **deploy:flectra_ubuntu**: Generate docker image tagged with git tag (=flectra version) and as latest (ubuntu based)
+- **deploy:flectra_debian**: Generate docker image tagged with git tag (=flectra version) and as latest (debian based)

setup/docker/base/debian/Dockerfile

@@ -0,0 +1,62 @@
+FROM debian:jessie
+
+LABEL maintainer="FlectraHQ <flectra@flectrahq.com>"
+LABEL authors="FlectraHQ,Jamotion"
+
+ARG POSTGRESQL_VERSION=9.6
+
+RUN apt-get update && \
+    apt-get install wget -y
+
+# adding postgresql debian repo in order to access all available postgresql version
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
+    echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+
+RUN apt-get update && \
+    apt-get install git gnupg2 python3 curl expect-dev node-less \
+    python3-pillow python3-lxml python-ldap3 python3-dev python3-pip python3-setuptools \
+    libpq5 libpq-dev postgresql-${POSTGRESQL_VERSION} postgresql-client-${POSTGRESQL_VERSION} \
+    postgresql-contrib-${POSTGRESQL_VERSION} postgresql-server-dev-${POSTGRESQL_VERSION} \
+    fontconfig libxfont1 xfonts-encodings xfonts-utils xfonts-base xfonts-75dpi \
+    npm nodejs libldap2-dev libsasl2-dev libxml2-dev libxslt1-dev libjpeg-dev zlib1g-dev libfontconfig1 -y
+
+RUN locale-gen --purge en_US.UTF-8 && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:UTF-8"\n' > /etc/default/locale
+
+ADD requirements.txt /tmp/requirements.txt
+
+RUN pip3 install --upgrade pip
+RUN pip3 install passlib
+RUN pip3 install -r /tmp/requirements.txt
+RUN npm install -g less less-plugin-clean-css -y && \
+    ln -s /usr/bin/nodejs /usr/bin/node
+
+RUN curl -o wkhtmltox.tar.xz -SL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \
+    && echo '3f923f425d345940089e44c1466f6408b9619562 wkhtmltox.tar.xz' | sha1sum -c - \
+    && tar xvf wkhtmltox.tar.xz \
+    && cp wkhtmltox/lib/* /usr/local/lib/ \
+    && cp wkhtmltox/bin/* /usr/local/bin/ \
+    && cp -r wkhtmltox/share/man/man1 /usr/local/share/man/
+
+
+# install CI tools
+RUN pip3 install coverage coveralls codecov simplejson pyserial pyyaml unittest2 pyopenssl>=16.2.0 && \
+    wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz && \
+    gunzip GeoLiteCity.dat.gz && \
+    mkdir -p /usr/share/GeoIP/ && \
+    mv GeoLiteCity.dat /usr/share/GeoIP/
+
+RUN addgroup --gid=5500 flectra && adduser --system --uid=5500 --gid=5500 --home=/opt/flectra --shell=/bin/bash flectra
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+USER flectra
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+USER root

setup/docker/base/ubuntu/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:bionic
+
+LABEL maintainer="FlectraHQ <flectra@flectrahq.com>"
+LABEL authors="FlectraHQ,Jamotion"
+
+ARG POSTGRESQL_VERSION=10
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install wget -y
+
+RUN apt-get update && \
+    apt-get install git gnupg2 python3 curl expect-dev node-less \
+    python3-pillow python3-lxml python-ldap3 python3-dev python3-pip python3-setuptools python3-passlib \
+    postgresql-${POSTGRESQL_VERSION} postgresql-server-dev-${POSTGRESQL_VERSION} \
+    fontconfig libxfont2 xfonts-encodings xfonts-utils xfonts-base xfonts-75dpi \
+    npm nodejs libldap2-dev libsasl2-dev libxml2-dev libxslt1-dev libjpeg-dev zlib1g-dev libfontconfig1 libmagickwand-dev -y
+
+RUN locale-gen --purge en_US.UTF-8 && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:UTF-8"\n' > /etc/default/locale
+
+ADD requirements.txt /tmp/requirements.txt
+
+RUN pip3 install --upgrade pip
+RUN pip3 install -r /tmp/requirements.txt
+RUN npm install -g less less-plugin-clean-css -y
+
+RUN curl -SsLO https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb && \
+    apt-get install -f ./wkhtmltox_0.12.5-1.bionic_amd64.deb
+
+# install CI tools
+RUN pip3 install coverage coveralls codecov simplejson pyserial pyyaml unittest2 pyopenssl>=16.2.0 \
+                 validate_email zpl2 unidecode shapely-geojson simplejson shapely geojson unidecode \
+                 email_validator wand imageio moviepy
+
+RUN addgroup --gid=5500 flectra && adduser --system --uid=5500 --gid=5500 --home=/opt/flectra --shell=/bin/bash flectra
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+USER flectra
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+USER root

setup/docker/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+# set the postgres database host, port, user and password according to the environment
+# and pass them as arguments to the flectra process if not present in the config file
+: ${HOST:=${DB_PORT_5432_TCP_ADDR:='db'}}
+: ${PORT:=${DB_PORT_5432_TCP_PORT:=5432}}
+: ${USER:=${DB_ENV_POSTGRES_USER:=${POSTGRES_USER:='flectra'}}}
+: ${PASSWORD:=${DB_ENV_POSTGRES_PASSWORD:=${POSTGRES_PASSWORD:='flectra'}}}
+
+DB_ARGS=()
+function check_config() {
+    param="$1"
+    value="$2"
+    DB_ARGS+=("--${param}")
+    DB_ARGS+=("${value}")
+}
+if [[ -z "${IGNORE_ENV}" ]]; then
+    check_config "db_host" "$HOST"
+    check_config "db_port" "$PORT"
+    check_config "db_user" "$USER"
+    check_config "db_password" "$PASSWORD"
+fi
+
+case "$1" in
+    -- | flectra)
+        shift
+        if [[ "$1" == "scaffold" ]] ; then
+            exec /opt/flectra/flectra-bin "$@"
+        else
+            exec /opt/flectra/flectra-bin "$@" "${DB_ARGS[@]}"
+        fi
+        ;;
+    -*)
+        exec /opt/flectra/flectra-bin "$@" "${DB_ARGS[@]}"
+        ;;
+    *)
+        exec "$@"
+esac
+
+exit 1

setup/docker/flectra.conf

@@ -0,0 +1,37 @@
+[options]
+addons_path = /mnt/extra-addons
+data_dir = /var/lib/flectra
+; admin_passwd = admin
+; csv_internal_sep = ,
+; db_maxconn = 64
+; db_name = False
+; db_template = template1
+; dbfilter = .*
+; debug_mode = False
+; email_from = False
+; limit_memory_hard = 2684354560
+; limit_memory_soft = 2147483648
+; limit_request = 8192
+; limit_time_cpu = 60
+; limit_time_real = 120
+; list_db = True
+; log_db = False
+; log_handler = [':INFO']
+; log_level = info
+; logfile = None
+; longpolling_port = 7072
+; max_cron_threads = 2
+; osv_memory_age_limit = 1.0
+; osv_memory_count_limit = False
+; smtp_password = False
+; smtp_port = 25
+; smtp_server = localhost
+; smtp_ssl = False
+; smtp_user = False
+; workers = 0
+; xmlrpc = True
+; xmlrpc_interface = 
+; xmlrpc_port = 7073
+; xmlrpcs = True
+; xmlrpcs_interface = 
+; xmlrpcs_port = 8071

setup/docker/flectra/dev/Dockerfile

@@ -0,0 +1,35 @@
+#Partially from Odoo's dockerfile
+#github.com/odoo/docker
+
+ARG BASE_IMAGE
+ARG BASE_TAG
+FROM ${BASE_IMAGE}:${BASE_TAG}
+LABEL maintainer="FlectraHQ <flectra@flectrahq.com>"
+LABEL authors="FlectraHQ,Jamotion"
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+USER root
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+# Install and configure OpenSSH Server (for development mode only)
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install openssh-server pwgen build-essential
+RUN mkdir -p /var/run/sshd \
+    && sed -i "s/UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config \
+    && sed -i "s/UsePAM.*/UsePAM no/g" /etc/ssh/sshd_config \
+    && sed -i "s/PermitRootLogin.*/PermitRootLogin yes/g" /etc/ssh/sshd_config
+
+# add special scripts for development
+ADD flectra/dev/bin /usr/local/bin
+
+VOLUME ["/opt/flectra", "/etc/flectra", "/var/log/flectra"]
+
+# Expose Flectra services
+EXPOSE 7072 7073 22
+
+ENTRYPOINT ["/usr/local/bin/run.sh"]

setup/docker/flectra/dev/bin/run.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+if [ "${AUTHORIZED_KEYS}" != "**None**" ]; then
+  echo "=> Found authorized keys"
+  mkdir -p /root/.ssh
+  chmod 700 /root/.ssh
+  touch /root/.ssh/authorized_keys
+  chmod 600 /root/.ssh/authorized_keys
+  IFS=$'\n'
+  arr=$(echo ${AUTHORIZED_KEYS} | tr "," "\n")
+  for x in $arr
+  do
+    x=$(echo $x |sed -e 's/^ *//' -e 's/ *$//')
+    cat /root/.ssh/authorized_keys | grep "$x" >/dev/null 2>&1
+    if [ $? -ne 0 ]; then
+      echo "=> Adding public key to /root/.ssh/authorized_keys: $x"
+      echo "$x" >> /root/.ssh/authorized_keys
+    fi
+  done
+fi
+
+/usr/local/bin/set_root_pw.sh
+exec /usr/sbin/sshd -D

setup/docker/flectra/dev/bin/set_root_pw.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+if [ -f /etc/flectra/ssh-pwd ]; then
+    echo "Root password already saved. Using this one"
+    PASS=$(</etc/flectra/ssh-pwd)
+else
+    PASS=${ROOT_PASS:-$(pwgen -s 12 1)}
+    _word=$( [ ${ROOT_PASS} ] && echo "preset" || echo "random" )
+    echo "=> Setting a ${_word} password to the root user"
+fi
+
+echo "root:$PASS" | chpasswd
+echo "flectra:$PASS" | chpasswd
+echo "=> Done!"
+echo $PASS > /etc/flectra/ssh-pwd
+IP=$(hostname --ip-address)
+echo "========================================================================"
+echo "     You can now connect to this Developer Container via SSH using:"
+echo ""
+echo "                      ssh -p <port> root@${IP}"
+echo "                              or"
+echo "                      ssh -p <port> flectra@${IP}"
+echo ""
+echo "            and enter the password '$PASS' when prompted"
+echo ""
+echo "========================================================================"

setup/docker/flectra/ubuntu/Dockerfile

@@ -0,0 +1,56 @@
+#Partially from Odoo's dockerfile
+#github.com/odoo/docker
+
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}:latest
+LABEL maintainer="FlectraHQ <flectra@flectrahq.com>"
+LABEL authors="FlectraHQ,Jamotion"
+
+ARG GITLAB_OWNER=flectra
+ARG GITLAB_COMMIT=master
+ARG ARG_DB_HOST=postgres
+ARG ARG_DB_PORT=5432
+ARG ARG_DB_USER=flectra
+ARG ARG_DB_PASSWORD=flectra
+
+ENV BRANCH=$ARG_BRANCH
+ENV HOST=$ARG_DB_HOST
+ENV PORT=$ARG_DB_PORT
+ENV USER=$ARG_DB_USER
+ENV PASSWORD=$ARG_DB_PASSWORD
+
+USER root
+RUN mkdir -p /var/log/flectra && \
+    chown -R flectra:flectra /var/log/flectra/
+
+RUN locale-gen --purge en_US.UTF-8 && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:en"\n' > /etc/default/locale
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+USER flectra
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+RUN cd /opt/flectra && \
+    wget -q -O flectra.tar.gz https://gitlab.com/${GITLAB_OWNER}/flectra/-/archive/${GITLAB_COMMIT}/flectra.tar.gz && \
+    tar -zxvf flectra.tar.gz flectra-${GITLAB_COMMIT}/flectra-bin flectra-${GITLAB_COMMIT}/flectra flectra-${GITLAB_COMMIT}/addons && \
+    cd flectra-${GITLAB_COMMIT} && \
+    mv addons/* flectra/addons/ && \
+    mv flectra ../. && \
+    mv flectra-bin ../. && \
+    cd /opt/flectra && \
+    rm -rf flectra-${GITLAB_COMMIT}
+
+EXPOSE 7073 7072
+
+ADD entrypoint.sh /opt/flectra/entrypoint.sh
+
+ENTRYPOINT ["/opt/flectra/entrypoint.sh"]
+CMD ["flectra"]
+

setup/docker/gitlab-ci/gitlab_after_sucesss

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# Part of code taken from github.com/OCA/maintainer-quality-tools
+
+import os
+
+from coverage.cmdline import main as coverage_main
+from coveralls import cli as coveralls_cli
+from codecov import main as codecov_main
+
+
+coverage_main(["report", "--show-missing"])
+try:
+    coveralls_cli.main(argv=None)
+except:
+    pass
+try:
+    codecov_main(argv=None)
+except:
+    pass

setup/docker/gitlab-ci/gitlab_test_flectra.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+
+# Part of code taken from github.com/OCA/maintainer-quality-tools
+
+import re
+import os
+import shutil
+import subprocess
+import sys
+import uuid
+import argparse
+from coverage.cmdline import main as coverage_main
+import locale
+
+try:
+    basestring
+except NameError:
+    basestring = str
+
+RED = "\033[1;31m"
+GREEN = "\033[1;32m"
+YELLOW = "\033[1;33m"
+YELLOW_LIGHT = "\033[33m"
+CLEAR = "\033[0;m"
+
+# modules that are either added as dependencies for other modules, so better to ignore them for the moment
+modules_to_ignore = ["account_analytic_default", "account_bank_statement_import", "account_budget",
+                     "account_cancel", "account_check_printing", "account_payment", "account_tax_python",
+                     "account_test", "account_voucher", "analytic", "anonymization", "association",
+                     "auth_crypt", "auth_ldap", "auth_oauth", "auth_signup", "barcodes", "base_address_city",
+                     "base_address_extended", "base_automation", "base_gengo", "base_geolocalize", "base_iban",
+                     "base_import", "base_import_module", "base_setup", "base_sparse_field", "base_vat",
+                     "base_vat_autocomplete", "bus", "calendar_sms", "crm_livechat",
+                     "crm_phone_validation", "crm_project", "decimal_precision", "delivery", "document", "event",
+                     "event_sale", "fetchmail", "gamification", "gamification_sale_crm", "google_account",
+                     "google_calendar", "google_drive", "google_spreadsheet",
+                     "hr_contract", "hr_expense_check", "hr_gamification", "hr_maintenance",
+                     "hr_org_chart", "hr_payroll", "hr_payroll_account", "hr_recruitment_survey",
+                     "hr_timesheet_attendance", "http_routing", "hw_blackbox_be", "hw_escpos", "hw_posbox_homepage",
+                     "hw_posbox_upgrade", "hw_proxy", "hw_scale", "hw_scanner", "hw_screen", "link_tracker",
+                     "live_currency_rate", "mass_mailing_event", "mass_mailing_event_track", "membership",
+                     "mrp_byproduct", "note_pad", "pad", "pad_project", "password_security",
+                     "payment", "payment_adyen", "payment_authorize", "payment_buckaroo", "payment_ogone",
+                     "payment_paypal", "payment_payumoney", "payment_sips", "payment_stripe", "payment_transfer",
+                     "phone_validation", "portal", "pos_cache", "pos_data_drinks", "pos_discount", "pos_mercury",
+                     "pos_reprint", "pos_restaurant", "pos_sale", "procurement_jit", "product", "product_email_template",
+                     "product_expiry", "product_extended", "product_margin", "project_timesheet_holidays", "rating",
+                     "rating_project", "report_intrastat", "resource", "sale_mrp", "sale_order_dates",
+                     "sale_payment", "sale_service_rating", "sale_timesheet", "sales_team", "sms", "stock_dropshipping",
+                     "stock_landed_costs", "stock_picking_batch", "survey_crm", "theme_bootswatch", "theme_default",
+                     "transifex", "utm", "web", "web_diagram", "web_editor", "web_kanban_gauge", "web_planner",
+                     "web_settings_dashboard", "web_tour", "website_crm", "website_crm_partner_assign",
+                     "website_crm_phone_validation", "website_customer", "website_event_questions",
+                     "website_event_sale", "website_event_track", "website_form", "website_form_project",
+                     "website_forum_doc", "website_gengo" , "website_google_map", "website_hr", "website_hr_recruitment",
+                     "website_links", "website_livechat", "website_mail", "website_mail_channel", "website_mass_mailing",
+                     "website_membership", "website_partner", "website_payment", "website_quote", "website_rating",
+                     "website_rating_project", "website_sale_comparison", "website_sale_delivery", "website_sale_digital",
+                     "website_sale_management", "website_sale_options", "website_sale_stock", "website_sale_stock_options",
+                     "website_sale_wishlist", "website_theme_install", "website_twitter", "theme_art", "theme_hermit",
+                     "theme_leith", "theme_techreceptives"]
+
+
+def colorized(text, color):
+    return '\n'.join(
+        map(lambda line: color + line + CLEAR, text.split('\n')))
+
+
+def green(text):
+    return colorized(text, GREEN)
+
+
+def yellow(text):
+    return colorized(text, YELLOW)
+
+
+def red(text):
+    return colorized(text, RED)
+
+
+def yellow_light(text):
+    return colorized(text, YELLOW_LIGHT)
+
+
+fail_msg = red("FAIL")
+success_msg = green("Success")
+
+
+def has_test_errors(fname, dbname, check_loaded=True):
+    """
+    Check a list of log lines for test errors.
+    Extension point to detect false positives.
+    """
+    # Rules defining checks to perform
+    # this can be
+    # - a string which will be checked in a simple substring match
+    # - a regex object that will be matched against the whole message
+    # - a callable that receives a dictionary of the form
+    #     {
+    #         'loglevel': ...,
+    #         'message': ....,
+    #     }
+    errors_ignore = [
+        'Mail delivery failed',
+        'failed sending mail',
+    ]
+    errors_report = [
+        lambda x: 'loglevel' in x and x['loglevel'] == 'CRITICAL',
+        'At least one test failed',
+        'no access rules, consider adding one',
+        'invalid module names, ignored',
+    ]
+
+    def make_pattern_list_callable(pattern_list):
+        for i in range(len(pattern_list)):
+            if isinstance(pattern_list[i], basestring):
+                regex = re.compile(pattern_list[i])
+                pattern_list[i] = lambda x, regex=regex: \
+                    regex.search(x['message'])
+            elif hasattr(pattern_list[i], 'match'):
+                regex = pattern_list[i]
+                pattern_list[i] = lambda x, regex=regex: \
+                    regex.search(x['message'])
+
+    make_pattern_list_callable(errors_ignore)
+    make_pattern_list_callable(errors_report)
+
+    print("-" * 10)
+    # Read log file removing ASCII color escapes:
+    # http://serverfault.com/questions/71285
+    color_regex = re.compile(r'\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]')
+    log_start_regex = re.compile(r'^.*(?P<loglevel>(INFO|WARNING|DEBUG|ERROR|CRITICAL)).*?: (?P<message>.*\S)\s*$')
+    log_records = []
+    last_log_record = dict.fromkeys(log_start_regex.groupindex.keys())
+    with open(fname, encoding='utf-8') as log:
+        for line in log:
+            line = color_regex.sub('', line)
+            if sys.stdout.encoding != 'UTF-8':
+                line = line.encode('ascii', 'ignore').decode('ascii')
+            match = log_start_regex.match(line)
+            if match:
+                last_log_record = match.groupdict()
+                log_records.append(last_log_record)
+            else:
+                last_log_record['message'] = '%s\n%s' % (
+                    last_log_record['message'], line.rstrip('\n')
+                )
+    errors = []
+    for log_record in log_records:
+        ignore = False
+        for ignore_pattern in errors_ignore:
+            if ignore_pattern(log_record):
+                ignore = True
+                break
+        if ignore:
+            break
+        print('{0}: {1}'.format(log_record['loglevel'], log_record['message']))
+        for report_pattern in errors_report:
+            if report_pattern(log_record):
+                errors.append(log_record)
+                break
+
+    if check_loaded:
+        if not [r for r in log_records if 'Modules loaded.' in r['message']]:
+            errors.append({'message': "Message not found: 'Modules loaded.'"})
+
+    if errors:
+        for e in errors:
+            print(e['message'])
+        print("-"*10)
+    return len(errors)
+
+
+def run_flectra(type_of_db, server_path, host, port, user, password):
+    print("\n creating database for : ", type_of_db)
+    try:
+        db = str(uuid.uuid1()) + "-" + type_of_db
+        
+        # Setting locale is needed because of gitlab runner server have
+        # other locales
+        os.environ["LANG"] = "en_US.UTF-8"
+        os.environ["LANGUAGE"] = "en_US:en"
+        os.environ["LC_ALL"] = "en_US.UTF-8"
+        
+        os.environ["PGPASSWORD"] = password
+        subprocess.check_call(
+                ["createdb", "-U", user, "-h", host, "-p", port, "-T",
+                 "template1", db])
+    except subprocess.CalledProcessError:
+        print("Problem in creating database.")
+    else:
+        log_file = os.path.join(os.getcwd(), "flectra.log")
+
+        # ugly hack as -i all is broken and does not take all modules into consideration
+        # need to remove l10n_* modules from the list as installing them all together
+        # creates trouble and build always fails
+        addons_path = os.path.join(server_path, "addons")
+        if not os.path.isdir(addons_path):
+            # Assume we are at integration test level where we can not test performance and client modules
+            addons_path = os.path.join(server_path, "flectra", "addons")
+            modules_to_ignore.append('test_performance')
+            modules_to_ignore.append('website')
+
+        addons = list(
+            set([addon.name for addon in os.scandir(addons_path) if addon.is_dir() and "10n" not in addon.name]) - set(
+                modules_to_ignore))
+
+        if type_of_db == "all":
+            modules_to_init = str(",".join(addons))
+        else:
+            modules_to_init = "base"
+
+        cmd_flectra = ["unbuffer", "coverage", "run", "--source", server_path]
+        cmd_flectra += ["%s/flectra-bin" % (server_path),
+                        "--db_host",
+                        host,
+                        "--db_port",
+                        port,
+                        "--db_user",
+                        user,
+                        "--db_password",
+                        password,
+                        "-d",
+                        db,
+                        '--db-filter',
+                        db,
+                        "--addons-path",
+                        addons_path,
+                        "--log-level=info",
+                        "--stop-after-init",
+                        "--test-enable",
+                        "--init", modules_to_init]
+        print("CMD EXECUTED --->>> ", " ".join(cmd_flectra))
+
+        with open(log_file, 'w') as outfile:
+            with subprocess.Popen(
+                    cmd_flectra,
+                    stdout=outfile,
+                    bufsize=1,
+                    universal_newlines=True) as p:
+                returncode = p.wait()
+        outfile.close()
+        errors = has_test_errors(os.path.join(server_path, log_file), db)
+    return {'errors': errors, 'returncode': returncode}
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description='Create Docker Instances....')
+    parser.add_argument('--build', dest='build', help='Port For Instance', required=True, default="base")
+    parser.add_argument('--server-path', dest='path', help='Flectra Path')
+    parser.add_argument('--host', dest='host',
+                        help='PostgreSQL Host',
+                        default="postgres")
+    parser.add_argument('--port', dest='port',
+                        help='PostgreSQL Port',
+                        default="5432")
+    parser.add_argument('--user', dest='user',
+                        help='PostgreSQL user',
+                        default="flectra")
+    parser.add_argument('--password', dest='password',
+                        help='PostgreSQL password',
+                        default="flectra")
+    args = parser.parse_args()
+    build = args.build
+
+    if argv is None:
+        argv = sys.argv
+    res = {}
+    server_path = args.path or os.getcwd()
+    
+    if build == "all":
+        res['all'] = run_flectra(
+                type_of_db="all",
+                server_path=server_path,
+                host=args.host,
+                port=args.port,
+                user=args.user,
+                password=args.password,
+        )
+        errors = 'all' in res and res['all'] and 'errors' in res['all'] and \
+                 res['all']['errors']
+    
+    if build == "base":
+        res['base'] = run_flectra(
+                type_of_db="base",
+                server_path=server_path,
+                host=args.host,
+                port=args.port,
+                user=args.user,
+                password=args.password,
+        )
+        errors = 'base' in res and res['base'] and 'errors' in res['base'] and \
+                 res['base']['errors']
+    
+    print("************ ERRORS ************  : ", errors)
+    return errors
+
+
+if __name__ == '__main__':
+    rc = main()
+    
+    if rc:
+        exit(rc)
+    
+    coverage_main(["report"])
+    
+    exit(rc)

setup/docker/requirements.txt

@@ -0,0 +1,49 @@
+Babel==2.3.4
+decorator==4.0.10
+docutils==0.12
+ebaysdk==2.1.5
+feedparser==5.2.1
+gevent==1.1.2 ; sys_platform != 'win32' and python_version < '3.7'
+gevent==1.3.4 ; sys_platform != 'win32' and python_version >= '3.7'
+greenlet==0.4.10 ; python_version < '3.7'
+greenlet==0.4.13 ; python_version >= '3.7'
+html2text==2016.9.19
+Jinja2==2.8.1
+lxml==3.7.1 ; sys_platform != 'win32' and python_version < '3.7'
+lxml==4.2.3 ; sys_platform != 'win32' and python_version >= '3.7'
+lxml ; sys_platform == 'win32'
+Mako==1.0.4
+MarkupSafe==0.23
+mock==2.0.0
+num2words==0.5.6
+oauthlib==2.0.7
+ofxparse==0.16
+passlib==1.6.5
+phonenumbers==8.8.9
+Pillow==4.0.0
+psutil==4.3.1; sys_platform != 'win32'
+psycopg2==2.7.3.1; sys_platform != 'win32'
+pydot==1.2.3
+pyldap==2.4.28; sys_platform != 'win32'
+pyparsing==2.1.10
+PyPDF2==1.26.0
+pyserial==3.1.1
+python-dateutil==2.5.3
+pytz==2016.7
+pyusb==1.0.0
+PyYAML==3.12 ; python_version < '3.7'
+PyYAML==3.13 ; python_version >= '3.7'
+qrcode==5.3
+reportlab==3.3.0
+requests==2.20.0
+suds-jurko==0.6
+vatnumber==1.2
+vobject==0.9.3
+Werkzeug==0.11.15
+XlsxWriter==0.9.3
+xlwt==1.3.*
+xlrd==1.0.0
+unittest2==1.1.0
+numpy==1.14.3
+pypiwin32 ; sys_platform == 'win32'
+pycrypto==2.6.1

setup/package.py

@@ -101,6 +101,258 @@ def publish(args, pub_type, extensions):
             published.append(_publish(release[0]))
     return published
 
+class FlectraDocker(object):
+    def __init__(self):
+        self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False)
+        self.port = 7073  # TODO sle: reliable way to get a free port?
+        self.prompt_re = '[root@nightly-tests] # '
+        self.timeout = 600
+
+    def system(self, command):
+        self.docker.sendline(command)
+        self.docker.expect_exact(self.prompt_re)
+
+    def start(self, docker_image, build_dir, pub_dir):
+        self.build_dir = build_dir
+        self.pub_dir = pub_dir
+
+        self.docker = pexpect.spawn(
+            'docker run -v %s:/opt/release -p 127.0.0.1:%s:7073'
+            ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image),
+            timeout=self.timeout,
+            searchwindowsize=len(self.prompt_re) + 1,
+        )
+        time.sleep(2)  # let the bash start
+        self.docker.logfile_read = self.log_file
+        self.id = subprocess.check_output('docker ps -l -q', shell=True).strip().decode('ascii')
+
+    def end(self):
+        try:
+            _rpc_count_modules(port=str(self.port))
+        except Exception as e:
+            logging.error('Exception during docker execution: %s:' % str(e))
+            logging.error('Error during docker execution: printing the bash output:')
+            with open(self.log_file.name) as f:
+                print('\n'.join(f.readlines()), file=stderr)
+            raise
+        finally:
+            self.docker.close()
+            system('docker rm -f %s' % self.id)
+            self.log_file.close()
+            os.remove(self.log_file.name)
+
+@contextmanager
+def docker(docker_image, build_dir, pub_dir):
+    _docker = FlectraDocker()
+    try:
+        _docker.start(docker_image, build_dir, pub_dir)
+        try:
+            yield _docker
+        except Exception:
+            raise
+    finally:
+        _docker.end()
+
+class KVM(object):
+    def __init__(self, o, image, ssh_key='', login='flectra'):
+        self.o = o
+        self.image = image
+        self.ssh_key = ssh_key
+        self.login = login
+
+    def timeout(self,signum,frame):
+        logging.warning("vm timeout kill",self.pid)
+        os.kill(self.pid,15)
+
+    def start(self):
+        l="kvm -cpu core2duo -smp 2,sockets=2,cores=1,threads=1 -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:17073-:7073,hostfwd=tcp:127.0.0.1:15432-:5432 -m 1024 -drive".split(" ")
+        #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
+        l.append('file=%s,snapshot=on'%self.image)
+        #l.extend(['-vnc','127.0.0.1:1'])
+        l.append('-nographic')
+        logging.info("Starting kvm: {}".format( " ".join(l)))
+        self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
+        time.sleep(50)
+        signal.alarm(2400)
+        signal.signal(signal.SIGALRM, self.timeout)
+        try:
+            self.run()
+        finally:
+            signal.signal(signal.SIGALRM, signal.SIG_DFL)
+            os.kill(self.pid,15)
+            time.sleep(10)
+
+    def ssh(self,cmd):
+        l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
+        system(l)
+
+    def rsync(self,args,options='--delete --exclude .git --exclude .tx --exclude __pycache__'):
+        cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
+        system(cmd)
+
+    def run(self):
+        pass
+
+class KVMWinBuildExe(KVM):
+    def run(self):
+        with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
+            f.write("VERSION=%s\n" % version.replace('~', '_').replace('+', ''))
+        with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
+            f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
+        with open(join(self.o.build_dir, 'setup/win32/Makefile.servicename'), 'w') as f:
+            f.write("SERVICENAME=%s\n" % nt_service_name)
+
+        remote_build_dir = '/cygdrive/c/flectrabuild/server/'
+
+        self.ssh("mkdir -p build")
+        logging.info("Syncing Flectra files to virtual machine...")
+        self.rsync('%s/ %s@127.0.0.1:%s' % (self.o.build_dir, self.login, remote_build_dir))
+        self.ssh("cd {}setup/win32;time make allinone;".format(remote_build_dir))
+        self.rsync('%s@127.0.0.1:%ssetup/win32/release/ %s/' % (self.login, remote_build_dir, self.o.build_dir), '')
+        logging.info("KVMWinBuildExe.run(): done")
+
+class KVMWinTestExe(KVM):
+    def run(self):
+        setuppath = glob("%s/flectra-server-setup-*.exe" % self.o.build_dir)[0]
+        setupfile = setuppath.split('/')[-1]
+        setupversion = setupfile.split('flectra-server-setup-')[1].split('.exe')[0]
+
+        self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login))
+        self.ssh("TEMP=/tmp ./%s /S" % setupfile)
+        self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Flectra %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
+        self.ssh('netsh advfirewall set publicprofile state off')
+        self.ssh('/cygdrive/c/"Program Files"/"Flectra {sv}"/python/python.exe \'c:\\Program Files\\Flectra {sv}\\server\\flectra-bin\' -d mycompany -i base --stop-after-init'.format(sv=setupversion))
+        _rpc_count_modules(port=17073)
+
+#----------------------------------------------------------
+# Stage: building
+#----------------------------------------------------------
+def _prepare_build_dir(o, win32=False):
+    
+    cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
+    if not win32:
+        cmd += ['--exclude', 'setup/win32']
+    system(cmd + ['%s/' % o.flectra_dir, o.build_dir])
+    for addon_path in glob(join(o.build_dir, 'addons/*')):
+        if addon_path.split(os.path.sep)[-1] not in ADDONS_NOT_TO_PUBLISH:
+            try:
+                shutil.move(addon_path, join(o.build_dir, 'flectra/addons'))
+            except shutil.Error as e:
+                # Thrown when the add-on is already in flectra/addons (if _prepare_build_dir
+                # has already been called once)
+                logging.warning("Warning '{}' while moving addon '{}'".format(e,addon_path))
+                if addon_path.startswith(o.build_dir) and os.path.isdir(addon_path):
+                    logging.info("Removing '{}'".format(addon_path))
+                    try:
+                        shutil.rmtree(addon_path)
+                    except shutil.Error as rm_error:
+                        logging.warning("Cannot remove '{}': {}".format(addon_path, rm_error))
+
+def build_tgz(o):
+    system(['python3', 'setup.py', 'sdist', '--quiet', '--formats=gztar,zip'], o.build_dir)
+    system(['mv', glob('%s/dist/flectra-*.tar.gz' % o.build_dir)[0], '%s/flectra_%s.%s.tar.gz' % (o.build_dir, version, timestamp)])
+    system(['mv', glob('%s/dist/flectra-*.zip' % o.build_dir)[0], '%s/flectra_%s.%s.zip' % (o.build_dir, version, timestamp)])
+
+def build_deb(o):
+    # Append timestamp to version for the .dsc to refer the right .tar.gz
+    cmd=['sed', '-i', '1s/^.*$/flectra (%s.%s) stable; urgency=low/'%(version,timestamp), 'debian/changelog']
+    subprocess.call(cmd, cwd=o.build_dir)
+    if not o.no_debsign:
+        deb = pexpect.spawn('dpkg-buildpackage -rfakeroot -k%s' % GPGID, cwd=o.build_dir)
+        deb.logfile = stdout.buffer
+        if GPGPASSPHRASE:
+            deb.expect_exact('Passphrase: ', timeout=1200)
+            deb.send(GPGPASSPHRASE + '\r\n')
+        deb.expect(pexpect.EOF, timeout=1200)
+    else:
+        subprocess.call(['dpkg-buildpackage', '-rfakeroot', '-uc', '-us'], cwd=o.build_dir)
+    # As the packages are built in the parent of the buildir, we move them back to build_dir
+    build_dir_parent = '{}/../'.format(o.build_dir)
+    wildcards = ['flectra_{}'.format(wc) for wc in ('*.deb', '*.dsc', '*_amd64.changes', '*.tar.gz', '*.tar.xz')]
+    move_glob(build_dir_parent, wildcards, o.build_dir)
+
+def build_rpm(o):
+    system(['python3', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir)
+    system(['mv', glob('%s/dist/flectra-*.noarch.rpm' % o.build_dir)[0], '%s/flectra_%s.%s.noarch.rpm' % (o.build_dir, version, timestamp)])
+
+def build_exe(o):
+    KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
+    system(['cp', glob('%s/flectra*.exe' % o.build_dir)[0], '%s/flectra_%s.%s.exe' % (o.build_dir, version, timestamp)])
+
+#----------------------------------------------------------
+# Stage: testing
+#----------------------------------------------------------
+def _prepare_testing(o):
+    logging.info('Preparing testing')
+    if not o.no_tarball:
+        logging.info('Preparing docker container instance for tarball')
+        subprocess.call(["mkdir", "docker_src"], cwd=o.build_dir)
+        subprocess.call(["cp", "package.dfsrc", os.path.join(o.build_dir, "docker_src", "Dockerfile")],
+                        cwd=os.path.join(o.flectra_dir, "setup"))
+        # Use rsync to copy requirements.txt in order to keep original permissions
+        subprocess.call(["rsync", "-a", "requirements.txt", os.path.join(o.build_dir, "docker_src")],
+                        cwd=os.path.join(o.flectra_dir))
+        subprocess.call(["docker", "build", "-t", "flectra-%s-src-nightly-tests" % docker_version, "."],
+                        cwd=os.path.join(o.build_dir, "docker_src"))
+    if not o.no_debian:
+        logging.info('Preparing docker container instance for debian')
+        subprocess.call(["mkdir", "docker_debian"], cwd=o.build_dir)
+        subprocess.call(["cp", "package.dfdebian", os.path.join(o.build_dir, "docker_debian", "Dockerfile")],
+                        cwd=os.path.join(o.flectra_dir, "setup"))
+        # Use rsync to copy requirements.txt in order to keep original permissions
+        subprocess.call(["rsync", "-a", "requirements.txt", os.path.join(o.build_dir, "docker_debian")],
+                        cwd=os.path.join(o.flectra_dir))
+        subprocess.call(["docker", "build", "-t", "flectra-%s-debian-nightly-tests" % docker_version, "."],
+                        cwd=os.path.join(o.build_dir, "docker_debian"))
+    if not o.no_rpm:
+        logging.info('Preparing docker container instance for RPM') 
+        subprocess.call(["mkdir", "docker_fedora"], cwd=o.build_dir)
+        subprocess.call(["cp", "package.dffedora", os.path.join(o.build_dir, "docker_fedora", "Dockerfile")],
+                        cwd=os.path.join(o.flectra_dir, "setup"))
+        subprocess.call(["docker", "build", "-t", "flectra-%s-fedora-nightly-tests" % docker_version, "."],
+                        cwd=os.path.join(o.build_dir, "docker_fedora"))
+
+def test_tgz(o):
+    logging.info('Testing tarball in docker')
+    with docker('flectra-%s-src-nightly-tests' % docker_version, o.build_dir, o.pub) as wheezy:
+        wheezy.release = '*.tar.gz'
+        wheezy.system("service postgresql start")
+        wheezy.system('pip3 install /opt/release/%s' % wheezy.release)
+        wheezy.system("useradd --system --no-create-home flectra")
+        wheezy.system('su postgres -s /bin/bash -c "createuser -s flectra"')
+        wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
+        wheezy.system('mkdir /var/lib/flectra')
+        wheezy.system('chown flectra:flectra /var/lib/flectra')
+        wheezy.system('su flectra -s /bin/bash -c "flectra -d mycompany -i base --stop-after-init"')
+        wheezy.system('su flectra -s /bin/bash -c "flectra -d mycompany &"')
+
+def test_deb(o):
+    logging.info('Testing deb package in docker')
+    with docker('flectra-%s-debian-nightly-tests' % docker_version, o.build_dir, o.pub) as wheezy:
+        wheezy.release = '*.deb'
+        wheezy.system("service postgresql start")
+        wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
+        wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release)
+        wheezy.system('/usr/bin/apt-get install -f -y')
+        wheezy.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -i base --stop-after-init"')
+        wheezy.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany &"')
+
+def test_rpm(o):
+    logging.info('Testing rpm in docker')
+    with docker('flectra-%s-fedora-nightly-tests' % docker_version, o.build_dir, o.pub) as fedora24:
+        fedora24.release = '*.noarch.rpm'
+        # Start postgresql
+        fedora24.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
+        fedora24.system('sleep 5')
+        fedora24.system('su postgres -c "createdb mycompany"')
+        # Flectra install
+        fedora24.system('dnf install -d 0 -e 0 /opt/release/%s -y' % fedora24.release)
+        fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -i base --stop-after-init"')
+        fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany &"')
+
+def test_exe(o):
+    logging.info('Testng windows installer in KVM')
+    KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 
 # ---------------------------------------------------------
 # Generates Packages, Sources and Release files of debian package