IntenseFinance/flectra
3
0
Fork 0
mirror of https://gitlab.com/flectra-hq/flectra.git synced 2025-02-25 18:55:21 -06:00
Code Issues Packages Projects Releases Wiki Activity

Dockerfile for ubuntu and debian base images created and started with

Browse Source 
ubuntu_flectra Dockerfile, but not yet finished)
This commit is contained in:
Renzo Meister
2019-10-29 06:22:14 +00:00
committed by Parthiv Patel
parent 6558e5d62f
commit 193fbd1f51
16 changed files with 1074 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -5,6 +5,7 @@ _build/
.*
!.gitignore
!.mailmap
!.gitlab-ci.yml
# compiled python files
*.py[co]
__pycache__/

293
.gitlab-ci.yml
View File

@@ -1,33 +1,270 @@
image: ubuntu:xenial
stages:
- test
job_test_base_pg_96:
stage: test
- unittest
- build
- integrationtest
- builddev
- deploy
- package
variables:
GITLAB_REGISTRY: "registry.gitlab.com/${CI_PROJECT_PATH}"
UBUNTU_BASE_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/base/ubuntu"
UBUNTU_FLECTRA_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/ubuntu"
UBUNTUDEV_FLECTRA_IMAGE: "registry.gitlab.com/${CI_PROJECT_PATH}/ubuntudev"
# DOCKER_HOST: tcp://docker:2375
# DOCKER_DRIVER: overlay2
# Configure postgres service (https://hub.docker.com/_/postgres/)
POSTGRES_DB: postgres
POSTGRES_USER: flectra
POSTGRES_PASSWORD: flectra
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
#--------------------------------------------------------------------
# Job Templates
#--------------------------------------------------------------------
.psql9:template:
services: &psql9_definition
- name: postgres:9-alpine
alias: postgres9
.psql10:template:
services: &psql10_definition
- name: postgres:10-alpine
alias: postgres10
.psql11:template:
services: &psql11_definition
- name: postgres:11-alpine
alias: postgres11
.unittest:template: &unittest_definition
image: ${UBUNTU_BASE_IMAGE}:latest
stage: unittest
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
paths:
- flectra.log
when: always
expire_in: 2 week
coverage: '/^TOTAL.*\s+(\d+\%)$/'
only:
- branches
except:
- schedules
.build:template: &build_definition
services:
- name: docker:dind
except:
- schedules
image: docker:stable
.integrationtest:template: &integrationtest_definition
stage: integrationtest
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
paths:
- flectra.log
when: always
expire_in: 2 week
only:
- tags
.build:base:template: &build_base_definition
stage: build
services:
- name: docker:dind
only:
- schedules
image: docker:stable
#--------------------------------------------------------------------
# Jobs of stage unittest
#--------------------------------------------------------------------
test:pg9_base:
<<: *unittest_definition
services: *psql9_definition
script:
- apt-get update && apt-get install -y apt-utils locales git
- git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH}
- gitlab_ci_install_flectra
- su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=base"
job_test_all_96:
stage: test
- cp -rp requirements.txt setup/docker/requirements.txt
- su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres9"
test:pg9_all:
<<: *unittest_definition
services: *psql9_definition
script:
- apt-get update && apt-get install -y apt-utils locales git
- git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH}
- gitlab_ci_install_flectra
- su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=all"
job_test_base_100:
stage: test
- cp -rp requirements.txt setup/docker/requirements.txt
- su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres9"
test:pg10_base:
<<: *unittest_definition
services: *psql10_definition
script:
- apt-get update && apt-get install -y apt-utils locales git
- export POSTGRESQL_VERSION=10
- git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH}
- gitlab_ci_install_flectra
- su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=base"
job_test_all_100:
stage: test
- cp -rp requirements.txt setup/docker/requirements.txt
- su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres10"
test:pg10_all:
<<: *unittest_definition
services: *psql10_definition
script:
- apt-get update && apt-get install -y apt-utils locales git
- export POSTGRESQL_VERSION=10
- git clone https://gitlab.com/flectra-hq/build-tools.git ${CI_PROJECT_DIR}/build-tools && export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH}
- gitlab_ci_install_flectra
- su - flectra -c "export PATH=${CI_PROJECT_DIR}/build-tools/gitlab-ci:${PATH} && gitlab_test_flectra --build=all"
- cp -rp requirements.txt setup/docker/requirements.txt
- su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres10"
test:pg11_base:
<<: *unittest_definition
services: *psql11_definition
script:
- cp -rp requirements.txt setup/docker/requirements.txt
- su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=base --host=postgres11"
test:pg11_all:
<<: *unittest_definition
services: *psql11_definition
script:
- cp -rp requirements.txt setup/docker/requirements.txt
- su - flectra -c "cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --host=postgres11"
#--------------------------------------------------------------------
# Jobs of stage build
#--------------------------------------------------------------------
build:flectra_ubuntu:
<<: *build_definition
stage: build
only:
- tags
script:
- echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
- docker pull ${UBUNTU_FLECTRA_IMAGE}:latest || true
- docker build --cache-from ${UBUNTU_FLECTRA_IMAGE}:latest
--build-arg BASE_IMAGE=${UBUNTU_BASE_IMAGE}
--build-arg GITLAB_OWNER=${CI_PROJECT_NAMESPACE}
--build-arg GITLAB_COMMIT=${CI_COMMIT_SHA}
--file ./setup/docker/flectra/ubuntu/Dockerfile
--tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
./setup/docker/
- docker push ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
#--------------------------------------------------------------------
# Jobs of stage integrationtest
#--------------------------------------------------------------------
itest:ubuntu_pg9_all:
<<: *integrationtest_definition
services: *psql9_definition
image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
script:
- cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres9
itest:ubuntu_pg10_all:
<<: *integrationtest_definition
services: *psql10_definition
image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
script:
- cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres10
itest:ubuntu_pg11_all:
<<: *integrationtest_definition
services: *psql11_definition
image: ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
script:
- cd ${CI_PROJECT_DIR} && ./setup/docker/gitlab-ci/gitlab_test_flectra.py --build=all --server-path=/opt/flectra --host=postgres11
#--------------------------------------------------------------------
# Jobs of stage builddev
#--------------------------------------------------------------------
build:flectra_ubuntu_dev:
<<: *build_definition
stage: builddev
only:
- tags
script:
- echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
- docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA || true
- docker build --cache-from ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
--build-arg BASE_IMAGE=${UBUNTU_FLECTRA_IMAGE}
--build-arg BASE_TAG=${CI_COMMIT_SHA}
--file ./setup/docker/flectra/dev/Dockerfile
--tag ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA
./setup/docker/
- docker push ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA
#--------------------------------------------------------------------
# Jobs of stage deploy
#--------------------------------------------------------------------
deploy:flectra_gitlab:
<<: *build_definition
stage: deploy
only:
- tags
script:
- echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
- docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
- docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_TAG
- docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTU_FLECTRA_IMAGE}:latest
- docker push ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_TAG
- docker push ${UBUNTU_FLECTRA_IMAGE}:latest
deploy:flectradev_gitlab:
<<: *build_definition
stage: deploy
only:
- tags
script:
- echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
- docker pull ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA
- docker tag ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_TAG
- docker tag ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${UBUNTUDEV_FLECTRA_IMAGE}:latest
- docker push ${UBUNTUDEV_FLECTRA_IMAGE}:$CI_COMMIT_TAG
- docker push ${UBUNTUDEV_FLECTRA_IMAGE}:latest
deploy:flectra_dockerhub:
<<: *build_definition
stage: deploy
only:
- tags
script:
- echo -n $DOCKER_HUB_PASSWORD | docker login -u $DOCKER_HUB_USER --password-stdin https://index.docker.io/v1/
- docker pull ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA
- docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${DOCKER_HUB_ROOT}/flectra:$CI_COMMIT_TAG
- docker tag ${UBUNTU_FLECTRA_IMAGE}:$CI_COMMIT_SHA ${DOCKER_HUB_ROOT}/flectra:latest
- docker push ${DOCKER_HUB_ROOT}/flectra:$CI_COMMIT_TAG
- docker push ${DOCKER_HUB_ROOT}/flectra:latest
.deploy:flectra_bin:
services:
- name: docker:dind
- name: postgres:10-alpine
alias: postgres
only:
- tags
image: docker:stable
stage: package
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
paths:
- build.log
when: always
expire_in: 1 day
script:
- apk add python3 python3-dev py-pip build-base rsync dpkg-dev rpm
- pip3 install pexpect
- mkdir /opt/flectra/
- mkdir /opt/flectra/log
- mkdir /opt/flectra/build
- mkdir /opt/flectra/public
- cd ${CI_PROJECT_DIR}/setup && python3 ./package.py -b /opt/flectra/build -p /opt/flectra/public --no-windows > ${CI_PROJECT_DIR}/build.log
- find /opt/flectra/public -maxdepth 3
#--------------------------------------------------------------------
# Scheduled jobs of stage build (executed by gitlab scheduler)
#--------------------------------------------------------------------
build:base_ubuntu:
<<: *build_base_definition
script:
- echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
- docker pull ${UBUNTU_BASE_IMAGE}:latest || true
- docker build --cache-from ${UBUNTU_BASE_IMAGE}:latest -f ./setup/docker/base/ubuntu/Dockerfile ./setup/docker/ --tag ${UBUNTU_BASE_IMAGE}:latest
- docker push ${UBUNTU_BASE_IMAGE}:latest

4
setup/docker/QUESTIONS.md Normal file
View File

@@ -0,0 +1,4 @@
Questions about docker integration
==================================
- how to avoid duplicate requirements.txt (e.g. setup/docker/requirements.txt)

94
setup/docker/README.md Normal file
View File

@@ -0,0 +1,94 @@
Continuous Integration and Docker Image Generation
==================================================
Intro
-----
This directory contains all the tools needed for optimized continuous integration testing
and direct deployment of docker images for development, test and production use.
On every push or merge into core flectra repository some gitlab-ci jobs will be executed
to test the changes and generate new docker images. The details are described in the
following chapters.
Goal
----
All code changes should be tested automatically in an acceptable duration and the revision
should be available with a few steps to run it as a docker container for manual testing.
Docker Image Inheritance
------------------------
There are some base docker images which are used for better performance
(and less waiting time for developers). Based on that images other images are generated.
The following image are deployed in gitlab registry for flectra:
| Image | Tag | Base Image | Trigger | Content
|--------------|-------------|--------------------|-----------|------------
| base/ubuntu | latest | ubuntu:xenial | scheduler | all required linux and python package to run flectra (ubuntu based)
| base/debian | latest | debian:jessie | scheduler | all required linux and python package to run flectra (debian based)
| ubuntu | **_build_** | base/ubuntu:latest | branch | full flectra installation and can be used for runbot like testing (ubuntu based)
| debian | **_build_** | base/debian:latest | branch | full flectra installation and can be used for runbot like testing (debian based)
| ubuntu | **_tag_** | base/ubuntu:latest | tag | full flectra installation for use in production (ubuntu based)
| debian | **_tag_** | base/debian:latest | tag | full flectra installation for use in production (debian based)
some additional images for testing and building are used as services:
- **postgres:9-alpine**: latest PostgreSQL 9.x image for running tests
- **postgres:10-alpine**: latest PostgreSQL 10.x image for running tests
- **docker:dind**: Special image containing docker to build and push images
Stages
------
The whole CI and docker process is split in four parts (called stages):
1. unittest: the jobs of this stage will validate the changes
2. build: at this stage the docker images are generated
3. integrationtest: these jobs will validate the flectra installation of docker images generated in previous stage
4. deploy: these jobs will deploy the docker images with flectra for production use (tagged wit git tag and as latest)
Only when all jobs of one stage are successful done, the next stage is started.
Jobs
----
### Stage: unittest
The following jobs are started on every push or merge of code:
- **test:pg9_base**: Run test of base module against PostgreSQL 9.x database
- **test:pg9_all**: Run test of all modules against PostgreSQL 9.x database
- **test:pg10_base**: Run test of base module against PostgreSQL 10.x database
- **test:pg9_all**: Run test of all modules against PostgreSQL 10.x database
### Stage: build
The following jobs are only started by gitlab scheduler:
-**build:base_ubuntu**: Build and push base ubuntu image containing all linux and python requirements
-**build:base_debian**: Build and push base debian image containing all linux and python requirements
The following jobs are started on push for a branch:
-**build:flectra_ubuntu**: Build and push image with installed flectra, tagged with commit id (ubuntu based)
-**build:flectra_debian**: Build and push image with installed flectra, tagged with commit id (debian based)
### Stage: integrationtest
The following jobs are started on push for a branch:
- **itest:ubuntu_pg9**: Run test of all modules against PostgreSQL 9.x database with flectra included in docker image (ubuntu based)
- **itest:ubuntu_pg10**: Run test of all modules against PostgreSQL 10.x database with flectra included in docker image (ubuntu based)
- **itest:debian_pg9**: Run test of all modules against PostgreSQL 9.x database with flectra included in docker image (debian based)
- **itest:debian_pg10**: Run test of all modules against PostgreSQL 10.x database with flectra included in docker image (debian based)
### Stage: deploy
The following jobs are started on push for tag:
- **deploy:flectra_ubuntu**: Generate docker image tagged with git tag (=flectra version) and as latest (ubuntu based)
- **deploy:flectra_debian**: Generate docker image tagged with git tag (=flectra version) and as latest (debian based)

62
setup/docker/base/debian/Dockerfile Normal file
View File

@@ -0,0 +1,62 @@
FROM debian:jessie
LABEL maintainer="FlectraHQ <flectra@flectrahq.com>"
LABEL authors="FlectraHQ,Jamotion"
ARG POSTGRESQL_VERSION=9.6
RUN apt-get update && \
apt-get install wget -y
# adding postgresql debian repo in order to access all available postgresql version
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" > /etc/apt/sources.list.d/pgdg.list
RUN apt-get update && \
apt-get install git gnupg2 python3 curl expect-dev node-less \
python3-pillow python3-lxml python-ldap3 python3-dev python3-pip python3-setuptools \
libpq5 libpq-dev postgresql-${POSTGRESQL_VERSION} postgresql-client-${POSTGRESQL_VERSION} \
postgresql-contrib-${POSTGRESQL_VERSION} postgresql-server-dev-${POSTGRESQL_VERSION} \
fontconfig libxfont1 xfonts-encodings xfonts-utils xfonts-base xfonts-75dpi \
npm nodejs libldap2-dev libsasl2-dev libxml2-dev libxslt1-dev libjpeg-dev zlib1g-dev libfontconfig1 -y
RUN locale-gen --purge en_US.UTF-8 && \
dpkg-reconfigure --frontend=noninteractive locales && \
echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:UTF-8"\n' > /etc/default/locale
ADD requirements.txt /tmp/requirements.txt
RUN pip3 install --upgrade pip
RUN pip3 install passlib
RUN pip3 install -r /tmp/requirements.txt
RUN npm install -g less less-plugin-clean-css -y && \
ln -s /usr/bin/nodejs /usr/bin/node
RUN curl -o wkhtmltox.tar.xz -SL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.4/wkhtmltox-0.12.4_linux-generic-amd64.tar.xz \
&& echo '3f923f425d345940089e44c1466f6408b9619562 wkhtmltox.tar.xz' | sha1sum -c - \
&& tar xvf wkhtmltox.tar.xz \
&& cp wkhtmltox/lib/* /usr/local/lib/ \
&& cp wkhtmltox/bin/* /usr/local/bin/ \
&& cp -r wkhtmltox/share/man/man1 /usr/local/share/man/
# install CI tools
RUN pip3 install coverage coveralls codecov simplejson pyserial pyyaml unittest2 pyopenssl>=16.2.0 && \
wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz && \
gunzip GeoLiteCity.dat.gz && \
mkdir -p /usr/share/GeoIP/ && \
mv GeoLiteCity.dat /usr/share/GeoIP/
RUN addgroup --gid=5500 flectra && adduser --system --uid=5500 --gid=5500 --home=/opt/flectra --shell=/bin/bash flectra
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
USER flectra
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
USER root

50
setup/docker/base/ubuntu/Dockerfile Normal file
View File

@@ -0,0 +1,50 @@
FROM ubuntu:bionic
LABEL maintainer="FlectraHQ <flectra@flectrahq.com>"
LABEL authors="FlectraHQ,Jamotion"
ARG POSTGRESQL_VERSION=10
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install wget -y
RUN apt-get update && \
apt-get install git gnupg2 python3 curl expect-dev node-less \
python3-pillow python3-lxml python-ldap3 python3-dev python3-pip python3-setuptools python3-passlib \
postgresql-${POSTGRESQL_VERSION} postgresql-server-dev-${POSTGRESQL_VERSION} \
fontconfig libxfont2 xfonts-encodings xfonts-utils xfonts-base xfonts-75dpi \
npm nodejs libldap2-dev libsasl2-dev libxml2-dev libxslt1-dev libjpeg-dev zlib1g-dev libfontconfig1 libmagickwand-dev -y
RUN locale-gen --purge en_US.UTF-8 && \
dpkg-reconfigure --frontend=noninteractive locales && \
echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:UTF-8"\n' > /etc/default/locale
ADD requirements.txt /tmp/requirements.txt
RUN pip3 install --upgrade pip
RUN pip3 install -r /tmp/requirements.txt
RUN npm install -g less less-plugin-clean-css -y
RUN curl -SsLO https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb && \
apt-get install -f ./wkhtmltox_0.12.5-1.bionic_amd64.deb
# install CI tools
RUN pip3 install coverage coveralls codecov simplejson pyserial pyyaml unittest2 pyopenssl>=16.2.0 \
validate_email zpl2 unidecode shapely-geojson simplejson shapely geojson unidecode \
email_validator wand imageio moviepy
RUN addgroup --gid=5500 flectra && adduser --system --uid=5500 --gid=5500 --home=/opt/flectra --shell=/bin/bash flectra
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
USER flectra
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
USER root

42
setup/docker/entrypoint.sh Executable file
View File

@@ -0,0 +1,42 @@
#!/bin/bash
set -e
# set the postgres database host, port, user and password according to the environment
# and pass them as arguments to the flectra process if not present in the config file
: ${HOST:=${DB_PORT_5432_TCP_ADDR:='db'}}
: ${PORT:=${DB_PORT_5432_TCP_PORT:=5432}}
: ${USER:=${DB_ENV_POSTGRES_USER:=${POSTGRES_USER:='flectra'}}}
: ${PASSWORD:=${DB_ENV_POSTGRES_PASSWORD:=${POSTGRES_PASSWORD:='flectra'}}}
DB_ARGS=()
function check_config() {
param="$1"
value="$2"
DB_ARGS+=("--${param}")
DB_ARGS+=("${value}")
}
if [[ -z "${IGNORE_ENV}" ]]; then
check_config "db_host" "$HOST"
check_config "db_port" "$PORT"
check_config "db_user" "$USER"
check_config "db_password" "$PASSWORD"
fi
case "$1" in
-- | flectra)
shift
if [[ "$1" == "scaffold" ]] ; then
exec /opt/flectra/flectra-bin "$@"
else
exec /opt/flectra/flectra-bin "$@" "${DB_ARGS[@]}"
fi
;;
-*)
exec /opt/flectra/flectra-bin "$@" "${DB_ARGS[@]}"
;;
*)
exec "$@"
esac
exit 1

37
setup/docker/flectra.conf Normal file
View File

@@ -0,0 +1,37 @@
[options]
addons_path = /mnt/extra-addons
data_dir = /var/lib/flectra
; admin_passwd = admin
; csv_internal_sep = ,
; db_maxconn = 64
; db_name = False
; db_template = template1
; dbfilter = .*
; debug_mode = False
; email_from = False
; limit_memory_hard = 2684354560
; limit_memory_soft = 2147483648
; limit_request = 8192
; limit_time_cpu = 60
; limit_time_real = 120
; list_db = True
; log_db = False
; log_handler = [':INFO']
; log_level = info
; logfile = None
; longpolling_port = 7072
; max_cron_threads = 2
; osv_memory_age_limit = 1.0
; osv_memory_count_limit = False
; smtp_password = False
; smtp_port = 25
; smtp_server = localhost
; smtp_ssl = False
; smtp_user = False
; workers = 0
; xmlrpc = True
; xmlrpc_interface =
; xmlrpc_port = 7073
; xmlrpcs = True
; xmlrpcs_interface =
; xmlrpcs_port = 8071

35
setup/docker/flectra/dev/Dockerfile Normal file
View File

@@ -0,0 +1,35 @@
#Partially from Odoo's dockerfile
#github.com/odoo/docker
ARG BASE_IMAGE
ARG BASE_TAG
FROM ${BASE_IMAGE}:${BASE_TAG}
LABEL maintainer="FlectraHQ <flectra@flectrahq.com>"
LABEL authors="FlectraHQ,Jamotion"
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
USER root
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
# Install and configure OpenSSH Server (for development mode only)
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install openssh-server pwgen build-essential
RUN mkdir -p /var/run/sshd \
&& sed -i "s/UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config \
&& sed -i "s/UsePAM.*/UsePAM no/g" /etc/ssh/sshd_config \
&& sed -i "s/PermitRootLogin.*/PermitRootLogin yes/g" /etc/ssh/sshd_config
# add special scripts for development
ADD flectra/dev/bin /usr/local/bin
VOLUME ["/opt/flectra", "/etc/flectra", "/var/log/flectra"]
# Expose Flectra services
EXPOSE 7072 7073 22
ENTRYPOINT ["/usr/local/bin/run.sh"]

22
setup/docker/flectra/dev/bin/run.sh Normal file
View File

@@ -0,0 +1,22 @@
#!/bin/bash
if [ "${AUTHORIZED_KEYS}" != "**None**" ]; then
echo "=> Found authorized keys"
mkdir -p /root/.ssh
chmod 700 /root/.ssh
touch /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
IFS=$'\n'
arr=$(echo ${AUTHORIZED_KEYS} | tr "," "\n")
for x in $arr
do
x=$(echo $x |sed -e 's/^ *//' -e 's/ *$//')
cat /root/.ssh/authorized_keys | grep "$x" >/dev/null 2>&1
if [ $? -ne 0 ]; then
echo "=> Adding public key to /root/.ssh/authorized_keys: $x"
echo "$x" >> /root/.ssh/authorized_keys
fi
done
fi
/usr/local/bin/set_root_pw.sh
exec /usr/sbin/sshd -D

25
setup/docker/flectra/dev/bin/set_root_pw.sh Normal file
View File

@@ -0,0 +1,25 @@
#!/bin/bash
if [ -f /etc/flectra/ssh-pwd ]; then
echo "Root password already saved. Using this one"
PASS=$(</etc/flectra/ssh-pwd)
else
PASS=${ROOT_PASS:-$(pwgen -s 12 1)}
_word=$( [ ${ROOT_PASS} ] && echo "preset" || echo "random" )
echo "=> Setting a ${_word} password to the root user"
fi
echo "root:$PASS" | chpasswd
echo "flectra:$PASS" | chpasswd
echo "=> Done!"
echo $PASS > /etc/flectra/ssh-pwd
IP=$(hostname --ip-address)
echo "========================================================================"
echo " You can now connect to this Developer Container via SSH using:"
echo ""
echo " ssh -p <port> root@${IP}"
echo " or"
echo " ssh -p <port> flectra@${IP}"
echo ""
echo " and enter the password '$PASS' when prompted"
echo ""
echo "========================================================================"

56
setup/docker/flectra/ubuntu/Dockerfile Normal file
View File

@@ -0,0 +1,56 @@
#Partially from Odoo's dockerfile
#github.com/odoo/docker
ARG BASE_IMAGE
FROM ${BASE_IMAGE}:latest
LABEL maintainer="FlectraHQ <flectra@flectrahq.com>"
LABEL authors="FlectraHQ,Jamotion"
ARG GITLAB_OWNER=flectra
ARG GITLAB_COMMIT=master
ARG ARG_DB_HOST=postgres
ARG ARG_DB_PORT=5432
ARG ARG_DB_USER=flectra
ARG ARG_DB_PASSWORD=flectra
ENV BRANCH=$ARG_BRANCH
ENV HOST=$ARG_DB_HOST
ENV PORT=$ARG_DB_PORT
ENV USER=$ARG_DB_USER
ENV PASSWORD=$ARG_DB_PASSWORD
USER root
RUN mkdir -p /var/log/flectra && \
chown -R flectra:flectra /var/log/flectra/
RUN locale-gen --purge en_US.UTF-8 && \
dpkg-reconfigure --frontend=noninteractive locales && \
echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:en"\n' > /etc/default/locale
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
USER flectra
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
RUN cd /opt/flectra && \
wget -q -O flectra.tar.gz https://gitlab.com/${GITLAB_OWNER}/flectra/-/archive/${GITLAB_COMMIT}/flectra.tar.gz && \
tar -zxvf flectra.tar.gz flectra-${GITLAB_COMMIT}/flectra-bin flectra-${GITLAB_COMMIT}/flectra flectra-${GITLAB_COMMIT}/addons && \
cd flectra-${GITLAB_COMMIT} && \
mv addons/* flectra/addons/ && \
mv flectra ../. && \
mv flectra-bin ../. && \
cd /opt/flectra && \
rm -rf flectra-${GITLAB_COMMIT}
EXPOSE 7073 7072
ADD entrypoint.sh /opt/flectra/entrypoint.sh
ENTRYPOINT ["/opt/flectra/entrypoint.sh"]
CMD ["flectra"]

20
setup/docker/gitlab-ci/gitlab_after_sucesss Executable file
View File

@@ -0,0 +1,20 @@
#!/usr/bin/env python
# Part of code taken from github.com/OCA/maintainer-quality-tools
import os
from coverage.cmdline import main as coverage_main
from coveralls import cli as coveralls_cli
from codecov import main as codecov_main
coverage_main(["report", "--show-missing"])
try:
coveralls_cli.main(argv=None)
except:
pass
try:
codecov_main(argv=None)
except:
pass

307
setup/docker/gitlab-ci/gitlab_test_flectra.py Executable file
View File

@@ -0,0 +1,307 @@
#!/usr/bin/env python3
# Part of code taken from github.com/OCA/maintainer-quality-tools
import re
import os
import shutil
import subprocess
import sys
import uuid
import argparse
from coverage.cmdline import main as coverage_main
import locale
try:
basestring
except NameError:
basestring = str
RED = "\033[1;31m"
GREEN = "\033[1;32m"
YELLOW = "\033[1;33m"
YELLOW_LIGHT = "\033[33m"
CLEAR = "\033[0;m"
# modules that are either added as dependencies for other modules, so better to ignore them for the moment
modules_to_ignore = ["account_analytic_default", "account_bank_statement_import", "account_budget",
"account_cancel", "account_check_printing", "account_payment", "account_tax_python",
"account_test", "account_voucher", "analytic", "anonymization", "association",
"auth_crypt", "auth_ldap", "auth_oauth", "auth_signup", "barcodes", "base_address_city",
"base_address_extended", "base_automation", "base_gengo", "base_geolocalize", "base_iban",
"base_import", "base_import_module", "base_setup", "base_sparse_field", "base_vat",
"base_vat_autocomplete", "bus", "calendar_sms", "crm_livechat",
"crm_phone_validation", "crm_project", "decimal_precision", "delivery", "document", "event",
"event_sale", "fetchmail", "gamification", "gamification_sale_crm", "google_account",
"google_calendar", "google_drive", "google_spreadsheet",
"hr_contract", "hr_expense_check", "hr_gamification", "hr_maintenance",
"hr_org_chart", "hr_payroll", "hr_payroll_account", "hr_recruitment_survey",
"hr_timesheet_attendance", "http_routing", "hw_blackbox_be", "hw_escpos", "hw_posbox_homepage",
"hw_posbox_upgrade", "hw_proxy", "hw_scale", "hw_scanner", "hw_screen", "link_tracker",
"live_currency_rate", "mass_mailing_event", "mass_mailing_event_track", "membership",
"mrp_byproduct", "note_pad", "pad", "pad_project", "password_security",
"payment", "payment_adyen", "payment_authorize", "payment_buckaroo", "payment_ogone",
"payment_paypal", "payment_payumoney", "payment_sips", "payment_stripe", "payment_transfer",
"phone_validation", "portal", "pos_cache", "pos_data_drinks", "pos_discount", "pos_mercury",
"pos_reprint", "pos_restaurant", "pos_sale", "procurement_jit", "product", "product_email_template",
"product_expiry", "product_extended", "product_margin", "project_timesheet_holidays", "rating",
"rating_project", "report_intrastat", "resource", "sale_mrp", "sale_order_dates",
"sale_payment", "sale_service_rating", "sale_timesheet", "sales_team", "sms", "stock_dropshipping",
"stock_landed_costs", "stock_picking_batch", "survey_crm", "theme_bootswatch", "theme_default",
"transifex", "utm", "web", "web_diagram", "web_editor", "web_kanban_gauge", "web_planner",
"web_settings_dashboard", "web_tour", "website_crm", "website_crm_partner_assign",
"website_crm_phone_validation", "website_customer", "website_event_questions",
"website_event_sale", "website_event_track", "website_form", "website_form_project",
"website_forum_doc", "website_gengo" , "website_google_map", "website_hr", "website_hr_recruitment",
"website_links", "website_livechat", "website_mail", "website_mail_channel", "website_mass_mailing",
"website_membership", "website_partner", "website_payment", "website_quote", "website_rating",
"website_rating_project", "website_sale_comparison", "website_sale_delivery", "website_sale_digital",
"website_sale_management", "website_sale_options", "website_sale_stock", "website_sale_stock_options",
"website_sale_wishlist", "website_theme_install", "website_twitter", "theme_art", "theme_hermit",
"theme_leith", "theme_techreceptives"]
def colorized(text, color):
return '\n'.join(
map(lambda line: color + line + CLEAR, text.split('\n')))
def green(text):
return colorized(text, GREEN)
def yellow(text):
return colorized(text, YELLOW)
def red(text):
return colorized(text, RED)
def yellow_light(text):
return colorized(text, YELLOW_LIGHT)
fail_msg = red("FAIL")
success_msg = green("Success")
def has_test_errors(fname, dbname, check_loaded=True):
"""
Check a list of log lines for test errors.
Extension point to detect false positives.
"""
# Rules defining checks to perform
# this can be
# - a string which will be checked in a simple substring match
# - a regex object that will be matched against the whole message
# - a callable that receives a dictionary of the form
# {
# 'loglevel': ...,
# 'message': ....,
# }
errors_ignore = [
'Mail delivery failed',
'failed sending mail',
]
errors_report = [
lambda x: 'loglevel' in x and x['loglevel'] == 'CRITICAL',
'At least one test failed',
'no access rules, consider adding one',
'invalid module names, ignored',
]
def make_pattern_list_callable(pattern_list):
for i in range(len(pattern_list)):
if isinstance(pattern_list[i], basestring):
regex = re.compile(pattern_list[i])
pattern_list[i] = lambda x, regex=regex: \
regex.search(x['message'])
elif hasattr(pattern_list[i], 'match'):
regex = pattern_list[i]
pattern_list[i] = lambda x, regex=regex: \
regex.search(x['message'])
make_pattern_list_callable(errors_ignore)
make_pattern_list_callable(errors_report)
print("-" * 10)
# Read log file removing ASCII color escapes:
# http://serverfault.com/questions/71285
color_regex = re.compile(r'\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]')
log_start_regex = re.compile(r'^.*(?P<loglevel>(INFO|WARNING|DEBUG|ERROR|CRITICAL)).*?: (?P<message>.*\S)\s*$')
log_records = []
last_log_record = dict.fromkeys(log_start_regex.groupindex.keys())
with open(fname, encoding='utf-8') as log:
for line in log:
line = color_regex.sub('', line)
if sys.stdout.encoding != 'UTF-8':
line = line.encode('ascii', 'ignore').decode('ascii')
match = log_start_regex.match(line)
if match:
last_log_record = match.groupdict()
log_records.append(last_log_record)
else:
last_log_record['message'] = '%s\n%s' % (
last_log_record['message'], line.rstrip('\n')
)
errors = []
for log_record in log_records:
ignore = False
for ignore_pattern in errors_ignore:
if ignore_pattern(log_record):
ignore = True
break
if ignore:
break
print('{0}: {1}'.format(log_record['loglevel'], log_record['message']))
for report_pattern in errors_report:
if report_pattern(log_record):
errors.append(log_record)
break
if check_loaded:
if not [r for r in log_records if 'Modules loaded.' in r['message']]:
errors.append({'message': "Message not found: 'Modules loaded.'"})
if errors:
for e in errors:
print(e['message'])
print("-"*10)
return len(errors)
def run_flectra(type_of_db, server_path, host, port, user, password):
print("\n creating database for : ", type_of_db)
try:
db = str(uuid.uuid1()) + "-" + type_of_db
# Setting locale is needed because of gitlab runner server have
# other locales
os.environ["LANG"] = "en_US.UTF-8"
os.environ["LANGUAGE"] = "en_US:en"
os.environ["LC_ALL"] = "en_US.UTF-8"
os.environ["PGPASSWORD"] = password
subprocess.check_call(
["createdb", "-U", user, "-h", host, "-p", port, "-T",
"template1", db])
except subprocess.CalledProcessError:
print("Problem in creating database.")
else:
log_file = os.path.join(os.getcwd(), "flectra.log")
# ugly hack as -i all is broken and does not take all modules into consideration
# need to remove l10n_* modules from the list as installing them all together
# creates trouble and build always fails
addons_path = os.path.join(server_path, "addons")
if not os.path.isdir(addons_path):
# Assume we are at integration test level where we can not test performance and client modules
addons_path = os.path.join(server_path, "flectra", "addons")
modules_to_ignore.append('test_performance')
modules_to_ignore.append('website')
addons = list(
set([addon.name for addon in os.scandir(addons_path) if addon.is_dir() and "10n" not in addon.name]) - set(
modules_to_ignore))
if type_of_db == "all":
modules_to_init = str(",".join(addons))
else:
modules_to_init = "base"
cmd_flectra = ["unbuffer", "coverage", "run", "--source", server_path]
cmd_flectra += ["%s/flectra-bin" % (server_path),
"--db_host",
host,
"--db_port",
port,
"--db_user",
user,
"--db_password",
password,
"-d",
db,
'--db-filter',
db,
"--addons-path",
addons_path,
"--log-level=info",
"--stop-after-init",
"--test-enable",
"--init", modules_to_init]
print("CMD EXECUTED --->>> ", " ".join(cmd_flectra))
with open(log_file, 'w') as outfile:
with subprocess.Popen(
cmd_flectra,
stdout=outfile,
bufsize=1,
universal_newlines=True) as p:
returncode = p.wait()
outfile.close()
errors = has_test_errors(os.path.join(server_path, log_file), db)
return {'errors': errors, 'returncode': returncode}
def main(argv=None):
parser = argparse.ArgumentParser(description='Create Docker Instances....')
parser.add_argument('--build', dest='build', help='Port For Instance', required=True, default="base")
parser.add_argument('--server-path', dest='path', help='Flectra Path')
parser.add_argument('--host', dest='host',
help='PostgreSQL Host',
default="postgres")
parser.add_argument('--port', dest='port',
help='PostgreSQL Port',
default="5432")
parser.add_argument('--user', dest='user',
help='PostgreSQL user',
default="flectra")
parser.add_argument('--password', dest='password',
help='PostgreSQL password',
default="flectra")
args = parser.parse_args()
build = args.build
if argv is None:
argv = sys.argv
res = {}
server_path = args.path or os.getcwd()
if build == "all":
res['all'] = run_flectra(
type_of_db="all",
server_path=server_path,
host=args.host,
port=args.port,
user=args.user,
password=args.password,
)
errors = 'all' in res and res['all'] and 'errors' in res['all'] and \
res['all']['errors']
if build == "base":
res['base'] = run_flectra(
type_of_db="base",
server_path=server_path,
host=args.host,
port=args.port,
user=args.user,
password=args.password,
)
errors = 'base' in res and res['base'] and 'errors' in res['base'] and \
res['base']['errors']
print("************ ERRORS ************ : ", errors)
return errors
if __name__ == '__main__':
rc = main()
if rc:
exit(rc)
coverage_main(["report"])
exit(rc)

49
setup/docker/requirements.txt Normal file
View File

@@ -0,0 +1,49 @@
Babel==2.3.4
decorator==4.0.10
docutils==0.12
ebaysdk==2.1.5
feedparser==5.2.1
gevent==1.1.2 ; sys_platform != 'win32' and python_version < '3.7'
gevent==1.3.4 ; sys_platform != 'win32' and python_version >= '3.7'
greenlet==0.4.10 ; python_version < '3.7'
greenlet==0.4.13 ; python_version >= '3.7'
html2text==2016.9.19
Jinja2==2.8.1
lxml==3.7.1 ; sys_platform != 'win32' and python_version < '3.7'
lxml==4.2.3 ; sys_platform != 'win32' and python_version >= '3.7'
lxml ; sys_platform == 'win32'
Mako==1.0.4
MarkupSafe==0.23
mock==2.0.0
num2words==0.5.6
oauthlib==2.0.7
ofxparse==0.16
passlib==1.6.5
phonenumbers==8.8.9
Pillow==4.0.0
psutil==4.3.1; sys_platform != 'win32'
psycopg2==2.7.3.1; sys_platform != 'win32'
pydot==1.2.3
pyldap==2.4.28; sys_platform != 'win32'
pyparsing==2.1.10
PyPDF2==1.26.0
pyserial==3.1.1
python-dateutil==2.5.3
pytz==2016.7
pyusb==1.0.0
PyYAML==3.12 ; python_version < '3.7'
PyYAML==3.13 ; python_version >= '3.7'
qrcode==5.3
reportlab==3.3.0
requests==2.20.0
suds-jurko==0.6
vatnumber==1.2
vobject==0.9.3
Werkzeug==0.11.15
XlsxWriter==0.9.3
xlwt==1.3.*
xlrd==1.0.0
unittest2==1.1.0
numpy==1.14.3
pypiwin32 ; sys_platform == 'win32'
pycrypto==2.6.1

10
setup/package.py
View File

@@ -362,13 +362,13 @@ def test_rpm(o):
with docker('flectra-%s-fedora-nightly-tests' % docker_version, o.build_dir, o.pub) as fedora24:
fedora24.release = '*.noarch.rpm'
# Start postgresql
fedora24.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
fedora24.system('sleep 5')
fedora24.system('su postgres -c "createdb mycompany"')
# fedora24.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
# fedora24.system('sleep 5')
# fedora24.system('su postgres -c "createdb mycompany"')
# Flectra install
fedora24.system('dnf install -d 0 -e 0 /opt/release/%s -y' % fedora24.release)
fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -i base --stop-after-init"')
fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany &"')
fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -i base --stop-after-init -r flectra -w flectra --db_host postgres --db_port 5432"')
fedora24.system('su flectra -s /bin/bash -c "flectra -c /etc/flectra/flectra.conf -d mycompany -r flectra -w flectra --db_host postgres --db_port 5432 &"')
def test_exe(o):
logging.info('Testng windows installer in KVM')