From fa646b01766dd3cfe9b7a3bf0ee2e62f100a6766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Laubacher?= Date: Tue, 5 May 2020 21:58:23 +0200 Subject: [PATCH 01/19] Add multi arch support --- .travis.yml | 3 +++ Dockerfile | 29 ++++++++++++++++++----------- data/infra/ci/install-docker.sh | 12 ++++++++++++ docker/build | 21 ++++++++++++++++----- 4 files changed, 49 insertions(+), 16 deletions(-) create mode 100755 data/infra/ci/install-docker.sh diff --git a/.travis.yml b/.travis.yml index a95bf67d..e4d43204 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +dist: bionic + language: php branches: @@ -15,6 +17,7 @@ cache: - $HOME/.composer/cache/files before_install: + - sudo ./data/infra/ci/install-docker.sh - sudo ./data/infra/ci/install-ms-odbc.sh - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - yes | pecl install pdo_sqlsrv swoole-4.4.15 diff --git a/Dockerfile b/Dockerfile index 64cd7ebe..edd8314a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,18 +23,25 @@ RUN \ apk add --no-cache libzip-dev zlib-dev libpng-dev && \ docker-php-ext-install -j"$(nproc)" zip gd -# Install swoole and sqlsrv driver -RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ - wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted mssql-tools_17.5.1.1-1_amd64.apk && \ - apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ - pecl install swoole-${SWOOLE_VERSION} pdo_sqlsrv && \ - docker-php-ext-enable swoole pdo_sqlsrv && \ - apk del .phpize-deps && \ - rm msodbcsql17_17.5.1.1-1_amd64.apk && \ - rm mssql-tools_17.5.1.1-1_amd64.apk +# Install sqlsrv driver +RUN if [ $(uname -m) == "x86_64" ]; then \ + wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ + wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.1.1-1_amd64.apk && \ + apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ + apk add --allow-untrusted mssql-tools_17.5.1.1-1_amd64.apk && \ + apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ + pecl install pdo_sqlsrv && \ + docker-php-ext-enable pdo_sqlsrv && \ + apk del .phpize-deps && \ + rm msodbcsql17_17.5.1.1-1_amd64.apk && \ + rm mssql-tools_17.5.1.1-1_amd64.apk ; \ + fi +# Install swoole +RUN apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS && \ + pecl install swoole-${SWOOLE_VERSION} && \ + docker-php-ext-enable swoole && \ + apk del .phpize-deps # Install shlink FROM base as builder diff --git a/data/infra/ci/install-docker.sh b/data/infra/ci/install-docker.sh new file mode 100755 index 00000000..58289a16 --- /dev/null +++ b/data/infra/ci/install-docker.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -ex + +# install latest docker version +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + +# enable multiarch execution +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes diff --git a/docker/build b/docker/build index 5eea7888..34ee7a21 100755 --- a/docker/build +++ b/docker/build @@ -1,15 +1,26 @@ #!/bin/bash set -e +BUILDX_VER=v0.4.1 +export DOCKER_CLI_EXPERIMENTAL=enabled + +mkdir -vp ~/.docker/cli-plugins/ ~/dockercache +curl --silent -L "https://github.com/docker/buildx/releases/download/${BUILDX_VER}/buildx-${BUILDX_VER}.linux-amd64" > ~/.docker/cli-plugins/docker-buildx +chmod a+x ~/.docker/cli-plugins/docker-buildx + +docker buildx create --use + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # If there is a tag, regardless the branch, build that docker tag and also "stable" if [[ ! -z $TRAVIS_TAG ]]; then - docker build --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} -t shlinkio/shlink:${TRAVIS_TAG#?} -t shlinkio/shlink:stable . - docker push shlinkio/shlink:${TRAVIS_TAG#?} - docker push shlinkio/shlink:stable + docker buildx build --push \ + --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} \ + --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ + -t shlinkio/shlink:${TRAVIS_TAG#?} -t shlinkio/shlink:stable . # If build branch is develop, build latest (on master, when there's no tag, do not build anything) elif [[ "$TRAVIS_BRANCH" == 'develop' ]]; then - docker build -t shlinkio/shlink:latest . - docker push shlinkio/shlink:latest + docker buildx build --push \ + --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ + -t shlinkio/shlink:latest . fi From 2ea58acde26f201992f9cdc7e676caffbb89e1c4 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 10:28:09 +0200 Subject: [PATCH 02/19] Updated changelog --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ede9184..64b529af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). +## [Unreleased] + +#### Added + +* [#709](https://github.com/shlinkio/shlink/issues/709) Added multi-architecture builds for the docker image. + +#### Changed + +* *Nothing* + +#### Deprecated + +* *Nothing* + +#### Removed + +* *Nothing* + +#### Fixed + +* *Nothing* + + ## 2.2.1 - 2020-05-11 #### Added From 135b62a9cc5aff171a72eca4fb58a6aa84823133 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 10:39:47 +0200 Subject: [PATCH 03/19] Documented multi-architecture on docker image --- Dockerfile | 1 + docker/README.md | 8 +++++++- docker/build | 8 ++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index a1b1f6f1..75c4ae2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,7 @@ RUN apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS && \ docker-php-ext-enable swoole && \ apk del .phpize-deps + # Install shlink FROM base as builder COPY . . diff --git a/docker/README.md b/docker/README.md index e17e570e..199dc6e2 100644 --- a/docker/README.md +++ b/docker/README.md @@ -263,7 +263,13 @@ Once created just run shlink with the volume: docker run --name shlink -p 8080:8080 -v ${PWD}/my/config/dir:/etc/shlink/config/params shlinkio/shlink:stable ``` -## Multi instance considerations +## Multi-architecture + +Starting on v2.3.0, Shlink's docker image is built for multiple architectures. + +The only limitation is that images for architectures other than `amd64` will not have support for Microsoft SQL databases, since there are no official binaries. + +## Multi-instance considerations These are some considerations to take into account when running multiple instances of shlink. diff --git a/docker/build b/docker/build index 22580702..4cc7f56a 100755 --- a/docker/build +++ b/docker/build @@ -19,13 +19,13 @@ if [[ ! -z $TRAVIS_TAG ]]; then [[ $TRAVIS_TAG != *"alpha"* && $TRAVIS_TAG != *"beta"* ]] && TAGS="${TAGS} -t shlinkio/shlink:stable" docker buildx build --push \ - --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} \ - --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ + --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} \ + --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ ${TAGS} . # If build branch is develop, build latest (on master, when there's no tag, do not build anything) elif [[ "$TRAVIS_BRANCH" == 'develop' ]]; then docker buildx build --push \ - --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ - -t shlinkio/shlink:latest . + --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ + -t shlinkio/shlink:latest . fi From 65e6676c0047b09a62fa2bf8b712c9043f7cd8d9 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 11:25:50 +0200 Subject: [PATCH 04/19] Removed docker image building on non-PR builds --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ce89e4bb..70ba9eb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,7 @@ before_script: script: - composer ci - - if [[ ! -z "$DOCKERFILE_CHANGED" && "${TRAVIS_PHP_VERSION}" == "7.4" ]]; then docker build -t shlink-docker-image:temp . ; fi + - if [[ ! -z "${DOCKERFILE_CHANGED}" && "${TRAVIS_PHP_VERSION}" == "7.4" && "${TRAVIS_PULL_REQUEST}" != 'false' ]]; then docker build -t shlink-docker-image:temp . ; fi after_success: - rm -f build/clover.xml @@ -47,7 +47,7 @@ after_success: # Before deploying, build dist file for current travis tag before_deploy: - rm -f ocular.phar - - if [[ ! -z $TRAVIS_TAG && "${TRAVIS_PHP_VERSION}" == "7.4" ]]; then ./build.sh ${TRAVIS_TAG#?} ; fi + - if [[ ! -z ${TRAVIS_TAG} && "${TRAVIS_PHP_VERSION}" == "7.4" ]]; then ./build.sh ${TRAVIS_TAG#?} ; fi deploy: - provider: releases @@ -62,5 +62,5 @@ deploy: script: bash ./docker/build on: all_branches: true - condition: $TRAVIS_PULL_REQUEST == 'false' + condition: "${TRAVIS_PULL_REQUEST}" == 'false' php: '7.4' From 09aa4cc977dee5a4f8b62664902497946e71db13 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 13:28:29 +0200 Subject: [PATCH 05/19] Changed travis build so that docker image publishing runs on its own separated job --- .travis.yml | 47 ++++++++++++++++++++++++++--------------------- docker/build | 1 + 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70ba9eb0..e65db293 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,17 @@ branches: only: - /.*/ -php: - - '7.4' +jobs: + include: + - name: "Docker publish" + php: '7.4' + if: env(TRAVIS_PULL_REQUEST) = 'false' + env: + - DOCKER_PUBLISH="true" + - name: "CI" + php: '7.4' + env: + - DOCKER_PUBLISH="false" services: - docker @@ -17,32 +26,33 @@ cache: - $HOME/.composer/cache/files before_install: - - sudo ./data/infra/ci/install-docker.sh - - sudo ./data/infra/ci/install-ms-odbc.sh - - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - - yes | pecl install pdo_sqlsrv swoole-4.4.18 - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0 + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then sudo ./data/infra/ci/install-docker.sh ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then sudo ./data/infra/ci/install-ms-odbc.sh ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then yes | pecl install pdo_sqlsrv swoole-4.4.18 ; fi install: - - composer self-update - - composer install --no-interaction --prefer-dist + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then composer self-update ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then composer install --no-interaction --prefer-dist ; fi before_script: - - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" ; fi - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) script: - - composer ci - - if [[ ! -z "${DOCKERFILE_CHANGED}" && "${TRAVIS_PHP_VERSION}" == "7.4" && "${TRAVIS_PULL_REQUEST}" != 'false' ]]; then docker build -t shlink-docker-image:temp . ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then composer ci ; fi + - if [[ ! -z "${DOCKERFILE_CHANGED}" && "${TRAVIS_PHP_VERSION}" == "7.4" && "${DOCKER_PUBLISH}" == "false" ]]; then docker build -t shlink-docker-image:temp . ; fi + - if [[ "${DOCKER_PUBLISH}" == 'true' ]]; then bash ./docker/build ; fi after_success: - rm -f build/clover.xml - - wget https://phar.phpunit.de/phpcov-7.0.2.phar - - phpdbg -qrr phpcov-7.0.2.phar merge build --clover build/clover.xml - - wget https://scrutinizer-ci.com/ocular.phar - - php ocular.phar code-coverage:upload --format=php-clover build/clover.xml + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then wget https://phar.phpunit.de/phpcov-7.0.2.phar ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then phpdbg -qrr phpcov-7.0.2.phar merge build --clover build/clover.xml ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then wget https://scrutinizer-ci.com/ocular.phar ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then php ocular.phar code-coverage:upload --format=php-clover build/clover.xml ; fi # Before deploying, build dist file for current travis tag before_deploy: @@ -56,11 +66,6 @@ deploy: file: "./build/shlink_${TRAVIS_TAG#?}_dist.zip" skip_cleanup: true on: + condition: "${DOCKER_PUBLISH}" == 'false' tags: true php: '7.4' - - provider: script - script: bash ./docker/build - on: - all_branches: true - condition: "${TRAVIS_PULL_REQUEST}" == 'false' - php: '7.4' diff --git a/docker/build b/docker/build index 4cc7f56a..141b0a8b 100755 --- a/docker/build +++ b/docker/build @@ -1,4 +1,5 @@ #!/bin/bash + set -e BUILDX_VER=v0.4.1 From 788f9635dd493c5c29454e9fa7a5787cb428fdbe Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 13:40:59 +0200 Subject: [PATCH 06/19] Fixed travis config syntax error --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e65db293..01c49ebb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,6 +66,7 @@ deploy: file: "./build/shlink_${TRAVIS_TAG#?}_dist.zip" skip_cleanup: true on: - condition: "${DOCKER_PUBLISH}" == 'false' + all_branches: true + condition: ${DOCKER_PUBLISH} == 'false' tags: true php: '7.4' From 489c739be2919f06e1ec7c12a7e92c2ebfa7ece1 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 14:00:03 +0200 Subject: [PATCH 07/19] Updated condition to run docker publish --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 01c49ebb..8459100b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ jobs: include: - name: "Docker publish" php: '7.4' - if: env(TRAVIS_PULL_REQUEST) = 'false' + if: NOT type = pull_request env: - DOCKER_PUBLISH="true" - name: "CI" From 75f5da5846dde52dee8924d61e14e76ee3fdb34d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 14:05:39 +0200 Subject: [PATCH 08/19] Fixed docker install in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8459100b..f2acb9ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ cache: before_install: - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0 - - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then sudo ./data/infra/ci/install-docker.sh ; fi + - if [[ "${DOCKER_PUBLISH}" == 'true' ]]; then sudo ./data/infra/ci/install-docker.sh ; fi - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then sudo ./data/infra/ci/install-ms-odbc.sh ; fi - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria ; fi - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then yes | pecl install pdo_sqlsrv swoole-4.4.18 ; fi From 726811f91ff36b129f0205dfbf757e372f0d97da Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 15:06:37 +0200 Subject: [PATCH 09/19] Separated docker builds in different platforms --- .travis.yml | 15 ++++++++++++++- docker/build | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2acb9ca..b3e8df21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,24 @@ branches: jobs: include: - - name: "Docker publish" + - name: "Docker publish - linux/arm/v7" php: '7.4' if: NOT type = pull_request env: - DOCKER_PUBLISH="true" + - PLATFORM="linux/arm/v7" + - name: "Docker publish - linux/arm64/v8" + php: '7.4' + if: NOT type = pull_request + env: + - DOCKER_PUBLISH="true" + - PLATFORM="linux/arm64/v8" + - name: "Docker publish - linux/amd64" + php: '7.4' + if: NOT type = pull_request + env: + - DOCKER_PUBLISH="true" + - PLATFORM="linux/amd64" - name: "CI" php: '7.4' env: diff --git a/docker/build b/docker/build index 141b0a8b..728bd216 100755 --- a/docker/build +++ b/docker/build @@ -21,12 +21,12 @@ if [[ ! -z $TRAVIS_TAG ]]; then docker buildx build --push \ --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} \ - --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ + --platform ${PLATFORM} \ ${TAGS} . # If build branch is develop, build latest (on master, when there's no tag, do not build anything) elif [[ "$TRAVIS_BRANCH" == 'develop' ]]; then docker buildx build --push \ - --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ + --platform ${PLATFORM} \ -t shlinkio/shlink:latest . fi From 17f3897746fd3335a896724a7bd8183b1656730d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 16 May 2020 22:01:20 +0200 Subject: [PATCH 10/19] Going back to single travis job for docker image building --- .travis.yml | 18 ++++-------------- docker/build | 4 ++-- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index b3e8df21..d9be39cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,29 +7,19 @@ branches: - /.*/ jobs: + fast_finish: true include: - - name: "Docker publish - linux/arm/v7" + - name: "Docker publish" php: '7.4' if: NOT type = pull_request env: - DOCKER_PUBLISH="true" - - PLATFORM="linux/arm/v7" - - name: "Docker publish - linux/arm64/v8" - php: '7.4' - if: NOT type = pull_request - env: - - DOCKER_PUBLISH="true" - - PLATFORM="linux/arm64/v8" - - name: "Docker publish - linux/amd64" - php: '7.4' - if: NOT type = pull_request - env: - - DOCKER_PUBLISH="true" - - PLATFORM="linux/amd64" - name: "CI" php: '7.4' env: - DOCKER_PUBLISH="false" + allow_failures: + - name: "Docker publish" services: - docker diff --git a/docker/build b/docker/build index 728bd216..141b0a8b 100755 --- a/docker/build +++ b/docker/build @@ -21,12 +21,12 @@ if [[ ! -z $TRAVIS_TAG ]]; then docker buildx build --push \ --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} \ - --platform ${PLATFORM} \ + --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ ${TAGS} . # If build branch is develop, build latest (on master, when there's no tag, do not build anything) elif [[ "$TRAVIS_BRANCH" == 'develop' ]]; then docker buildx build --push \ - --platform ${PLATFORM} \ + --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ -t shlinkio/shlink:latest . fi From f340e0e76e0910da7b5e983477138d4e7f15feab Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 17 May 2020 09:37:05 +0200 Subject: [PATCH 11/19] Temporary disabled ARM docker images to reduce build times --- docker/build | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/build b/docker/build index 141b0a8b..5b639d50 100755 --- a/docker/build +++ b/docker/build @@ -2,6 +2,8 @@ set -e +# PLATFORMS="linux/arm/v7,linux/arm64/v8,linux/amd64" +PLATFORMS="linux/amd64" BUILDX_VER=v0.4.1 export DOCKER_CLI_EXPERIMENTAL=enabled @@ -21,12 +23,12 @@ if [[ ! -z $TRAVIS_TAG ]]; then docker buildx build --push \ --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} \ - --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ + --platform ${PLATFORMS} \ ${TAGS} . # If build branch is develop, build latest (on master, when there's no tag, do not build anything) elif [[ "$TRAVIS_BRANCH" == 'develop' ]]; then docker buildx build --push \ - --platform linux/arm/v7,linux/arm64/v8,linux/amd64 \ + --platform ${PLATFORMS} \ -t shlinkio/shlink:latest . fi From 95ae5407990704d779ab72d63f1a87a5e6053b19 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 17 May 2020 10:19:54 +0200 Subject: [PATCH 12/19] Defined docker image to build in a var --- .travis.yml | 2 +- docker/build | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d9be39cd..8365f582 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ cache: before_install: - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0 - - if [[ "${DOCKER_PUBLISH}" == 'true' ]]; then sudo ./data/infra/ci/install-docker.sh ; fi - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then sudo ./data/infra/ci/install-ms-odbc.sh ; fi - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria ; fi - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then yes | pecl install pdo_sqlsrv swoole-4.4.18 ; fi install: + - if [[ "${DOCKER_PUBLISH}" == 'true' ]]; then sudo ./data/infra/ci/install-docker.sh ; fi - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then composer self-update ; fi - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then composer install --no-interaction --prefer-dist ; fi diff --git a/docker/build b/docker/build index 5b639d50..8ac27d4d 100755 --- a/docker/build +++ b/docker/build @@ -4,6 +4,7 @@ set -e # PLATFORMS="linux/arm/v7,linux/arm64/v8,linux/amd64" PLATFORMS="linux/amd64" +DOCKER_IMAGE="shlinkio/shlink" BUILDX_VER=v0.4.1 export DOCKER_CLI_EXPERIMENTAL=enabled @@ -17,9 +18,9 @@ echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # If there is a tag, regardless the branch, build that docker tag and also "stable" if [[ ! -z $TRAVIS_TAG ]]; then - TAGS="-t shlinkio/shlink:${TRAVIS_TAG#?}" + TAGS="-t ${DOCKER_IMAGE}:${TRAVIS_TAG#?}" # Push stable tag only if this is not an alpha or beta tag - [[ $TRAVIS_TAG != *"alpha"* && $TRAVIS_TAG != *"beta"* ]] && TAGS="${TAGS} -t shlinkio/shlink:stable" + [[ $TRAVIS_TAG != *"alpha"* && $TRAVIS_TAG != *"beta"* ]] && TAGS="${TAGS} -t ${DOCKER_IMAGE}:stable" docker buildx build --push \ --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} \ @@ -30,5 +31,5 @@ if [[ ! -z $TRAVIS_TAG ]]; then elif [[ "$TRAVIS_BRANCH" == 'develop' ]]; then docker buildx build --push \ --platform ${PLATFORMS} \ - -t shlinkio/shlink:latest . + -t ${DOCKER_IMAGE}:latest . fi From 371f246c419942033b6f41a66f2b7d1fc3eda2e8 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 8 Jun 2020 18:08:46 +0200 Subject: [PATCH 13/19] Improved custom slug sluggification, allowing valid URL characters --- composer.json | 11 +++++--- .../src/Util/CocurSymfonySluggerBridge.php | 26 +++++++++++++++++++ .../Validation/ShortUrlMetaInputFilter.php | 7 ++++- module/Core/test/Model/ShortUrlMetaTest.php | 20 +++++++++++--- 4 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 module/Core/src/Util/CocurSymfonySluggerBridge.php diff --git a/composer.json b/composer.json index 5d531a37..7ba81442 100644 --- a/composer.json +++ b/composer.json @@ -17,6 +17,7 @@ "ext-pdo": "*", "akrabat/ip-address-middleware": "^1.0", "cakephp/chronos": "^1.2", + "cocur/slugify": "^4.0", "doctrine/cache": "^1.9", "doctrine/dbal": "^2.10", "doctrine/migrations": "^2.2", @@ -53,11 +54,13 @@ "shlinkio/shlink-event-dispatcher": "^1.4", "shlinkio/shlink-installer": "^5.0.0", "shlinkio/shlink-ip-geolocation": "^1.4", - "symfony/console": "^5.0", - "symfony/filesystem": "^5.0", - "symfony/lock": "^5.0", + "symfony/console": "^5.1", + "symfony/filesystem": "^5.1", + "symfony/lock": "^5.1", "symfony/mercure": "^0.3.0", - "symfony/process": "^5.0" + "symfony/process": "~5.0.9", + "symfony/string": "^5.1", + "symfony/translation-contracts": "^2.1" }, "require-dev": { "devster/ubench": "^2.0", diff --git a/module/Core/src/Util/CocurSymfonySluggerBridge.php b/module/Core/src/Util/CocurSymfonySluggerBridge.php new file mode 100644 index 00000000..9415e47c --- /dev/null +++ b/module/Core/src/Util/CocurSymfonySluggerBridge.php @@ -0,0 +1,26 @@ +slugger = $slugger; + } + + public function slug(string $string, string $separator = '-', ?string $locale = null): AbstractUnicodeString + { + return s($this->slugger->slugify($string, $separator)); + } +} diff --git a/module/Core/src/Validation/ShortUrlMetaInputFilter.php b/module/Core/src/Validation/ShortUrlMetaInputFilter.php index 8fde5e98..4503117c 100644 --- a/module/Core/src/Validation/ShortUrlMetaInputFilter.php +++ b/module/Core/src/Validation/ShortUrlMetaInputFilter.php @@ -4,11 +4,13 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Validation; +use Cocur\Slugify\Slugify; use DateTime; use Laminas\InputFilter\Input; use Laminas\InputFilter\InputFilter; use Laminas\Validator; use Shlinkio\Shlink\Common\Validation; +use Shlinkio\Shlink\Core\Util\CocurSymfonySluggerBridge; use const Shlinkio\Shlink\Core\MIN_SHORT_CODES_LENGTH; @@ -46,7 +48,10 @@ class ShortUrlMetaInputFilter extends InputFilter // FIXME The only way to enforce the NotEmpty validator to be evaluated when the value is provided but it's // empty, is by using the deprecated setContinueIfEmpty $customSlug = $this->createInput(self::CUSTOM_SLUG, false)->setContinueIfEmpty(true); - $customSlug->getFilterChain()->attach(new Validation\SluggerFilter()); + $customSlug->getFilterChain()->attach(new Validation\SluggerFilter(new CocurSymfonySluggerBridge(new Slugify([ + 'regexp' => '/[^A-Za-z0-9._~]+/', + 'lowercase' => false, + ])))); $customSlug->getValidatorChain()->attach(new Validator\NotEmpty([ Validator\NotEmpty::STRING, Validator\NotEmpty::SPACE, diff --git a/module/Core/test/Model/ShortUrlMetaTest.php b/module/Core/test/Model/ShortUrlMetaTest.php index 7d0dd9b6..fe3d42fc 100644 --- a/module/Core/test/Model/ShortUrlMetaTest.php +++ b/module/Core/test/Model/ShortUrlMetaTest.php @@ -58,11 +58,14 @@ class ShortUrlMetaTest extends TestCase ]]; } - /** @test */ - public function properlyCreatedInstanceReturnsValues(): void + /** + * @test + * @dataProvider provideCustomSlugs + */ + public function properlyCreatedInstanceReturnsValues(string $customSlug, string $expectedSlug): void { $meta = ShortUrlMeta::fromRawData( - ['validSince' => Chronos::parse('2015-01-01')->toAtomString(), 'customSlug' => 'foobar'], + ['validSince' => Chronos::parse('2015-01-01')->toAtomString(), 'customSlug' => $customSlug], ); $this->assertTrue($meta->hasValidSince()); @@ -72,9 +75,18 @@ class ShortUrlMetaTest extends TestCase $this->assertNull($meta->getValidUntil()); $this->assertTrue($meta->hasCustomSlug()); - $this->assertEquals('foobar', $meta->getCustomSlug()); + $this->assertEquals($expectedSlug, $meta->getCustomSlug()); $this->assertFalse($meta->hasMaxVisits()); $this->assertNull($meta->getMaxVisits()); } + + public function provideCustomSlugs(): iterable + { + yield ['foobar', 'foobar']; + yield ['foo bar', 'foo-bar']; + yield ['wp-admin.php', 'wp-admin.php']; + yield ['UPPER_lower', 'UPPER_lower']; + yield ['more~url_special.chars', 'more~url_special.chars']; + } } From f274cafa7c5b3f04f5ef1b8116f8d3d77ed3d693 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 8 Jun 2020 18:10:34 +0200 Subject: [PATCH 14/19] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b529af..f61d06c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this #### Fixed -* *Nothing* +* [#769](https://github.com/shlinkio/shlink/issues/769) Fixed custom slugs not allowing valid URL characters, like `.`, `_` or `~`. ## 2.2.1 - 2020-05-11 From 527faf27a88f78cbfec71852183f95229a0172d8 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 7 Jun 2020 20:39:51 +0200 Subject: [PATCH 15/19] Changed how visits for a tag are fetched, avoiding thousands of values to be loaded in memory --- .../Core/src/Repository/VisitRepository.php | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index b3761c9f..5d242241 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -142,26 +142,18 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa private function createVisitsByTagQueryBuilder(string $tag, ?DateRange $dateRange = null): QueryBuilder { - $qb = $this->getEntityManager()->createQueryBuilder(); - $qb->select('s.id') - ->from(ShortUrl::class, 's') - ->join('s.tags', 't') - ->where($qb->expr()->eq('t.name', ':tag')) - ->setParameter('tag', $tag); - - $shortUrlIds = array_column($qb->getQuery()->getArrayResult(), 'id'); - $shortUrlIds[] = '-1'; // Add an invalid ID, in case the list is empty - // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later // Since they are not strictly provided by the caller, it's reasonably safe - $qb2 = $this->getEntityManager()->createQueryBuilder(); - $qb2->from(Visit::class, 'v') - ->where($qb2->expr()->in('v.shortUrl', $shortUrlIds)); + $qb = $this->getEntityManager()->createQueryBuilder(); + $qb->from(Visit::class, 'v') + ->join('v.shortUrl', 's') + ->join('s.tags', 't') + ->where($qb->expr()->eq('t.name', '\'' . $tag . '\'')); // Apply date range filtering - $this->applyDatesInline($qb2, $dateRange); + $this->applyDatesInline($qb, $dateRange); - return $qb2; + return $qb; } private function applyDatesInline(QueryBuilder $qb, ?DateRange $dateRange): void From 296134078cc8f9c4c550b406548773b84083703f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 7 Jun 2020 20:40:47 +0200 Subject: [PATCH 16/19] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f61d06c9..dcf0b004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this #### Fixed * [#769](https://github.com/shlinkio/shlink/issues/769) Fixed custom slugs not allowing valid URL characters, like `.`, `_` or `~`. +* [#781](https://github.com/shlinkio/shlink/issues/781) Fixed memory leak when loading visits for a tag which is used for big amounts of short URLs. ## 2.2.1 - 2020-05-11 From f3f3ef5c18dc95d298c2637654095afa412bbbb5 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 7 Jun 2020 20:42:44 +0200 Subject: [PATCH 17/19] Removed unused import --- module/Core/src/Repository/VisitRepository.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 5d242241..458b8ef2 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -12,8 +12,6 @@ use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; -use function array_column; - use const PHP_INT_MAX; class VisitRepository extends EntityRepository implements VisitRepositoryInterface From a4eda9d7616d26b9592403984992bc58a4ab3d1a Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 8 Jun 2020 22:38:51 +0200 Subject: [PATCH 18/19] Moved execution of API tests outside composer script --- .travis.yml | 2 +- composer.json | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8365f582..9b1c35e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ before_script: - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) script: - - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then composer ci ; fi + - if [[ "${DOCKER_PUBLISH}" == 'false' ]]; then bin/test/run-api-tests.sh --coverage-php build/coverage-api.cov && composer ci ; fi - if [[ ! -z "${DOCKERFILE_CHANGED}" && "${TRAVIS_PHP_VERSION}" == "7.4" && "${DOCKER_PUBLISH}" == "false" ]]; then docker build -t shlink-docker-image:temp . ; fi - if [[ "${DOCKER_PUBLISH}" == 'true' ]]; then bash ./docker/build ; fi diff --git a/composer.json b/composer.json index 7ba81442..ef9e0e92 100644 --- a/composer.json +++ b/composer.json @@ -58,7 +58,7 @@ "symfony/filesystem": "^5.1", "symfony/lock": "^5.1", "symfony/mercure": "^0.3.0", - "symfony/process": "~5.0.9", + "symfony/process": "^5.1", "symfony/string": "^5.1", "symfony/translation-contracts": "^2.1" }, @@ -112,8 +112,7 @@ ], "test:ci": [ "@test:unit:ci", - "@test:db", - "@test:api:ci" + "@test:db" ], "test:unit": "phpdbg -qrr vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox", "test:unit:ci": "@test:unit --coverage-clover=build/clover.xml --coverage-xml=build/coverage-xml --log-junit=build/junit.xml", From 8ecc9c69a24d2e0872ef771c784a21d19309d61d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 8 Jun 2020 22:49:40 +0200 Subject: [PATCH 19/19] Added v2.2.2 to changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcf0b004..6192481f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). -## [Unreleased] +## 2.2.2 - 2020-06-08 #### Added