Merge pull request #1832 from CeetronSolutions/python-pypi-cleanup

Python pypi package work

CMakeLists.txt

@@ -255,10 +255,11 @@ if (OPM_ENABLE_PYTHON)
   list(APPEND _opmcommon_include_dirs ${_ecl_include_dirs})
   string(REPLACE ";" ":" _setup_include_dirs "${_opmcommon_include_dirs}")
 
+  get_target_property(_opmcommon_lib_dirs opmcommon LINK_DIRECTORIES)
   if (CMAKE_PREFIX_PATH)
-    set(_opmcommon_lib_dirs ${PROJECT_BINARY_DIR}/lib ${CMAKE_PREFIX_PATH}/${CMAKE_INSTALL_LIBDIR})
+    list(APPEND _opmcommon_lib_dirs ${PROJECT_BINARY_DIR}/lib ${CMAKE_PREFIX_PATH}/${CMAKE_INSTALL_LIBDIR})
   else()
-    set(_opmcommon_lib_dirs ${PROJECT_BINARY_DIR}/lib)
+    list(APPEND _opmcommon_lib_dirs ${PROJECT_BINARY_DIR}/lib)
   endif()
   string(REPLACE ";" ":" _setup_lib_dirs "${_opmcommon_lib_dirs}")
 
@@ -284,6 +285,11 @@ if (OPM_ENABLE_PYTHON)
     set( _rpath_arg "")
   endif()
 
+  set(opm-common_PYTHON_PACKAGE_VERSION ${OPM_PYTHON_PACKAGE_VERSION_TAG})
+
+  # Generate versioned setup.py
+  configure_file (${PROJECT_SOURCE_DIR}/python/setup.py.in ${PROJECT_BINARY_DIR}/python/setup.py)
+  file(COPY ${PROJECT_SOURCE_DIR}/python/README.md DESTINATION ${PROJECT_BINARY_DIR}/python)
   execute_process(COMMAND ${PYTHON_EXECUTABLE} target_name.py
                   WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python
                   OUTPUT_VARIABLE python_lib_target)
@@ -291,19 +297,19 @@ if (OPM_ENABLE_PYTHON)
   add_custom_target(copy_python ALL
     COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/python/install.py ${PROJECT_SOURCE_DIR}/python ${PROJECT_BINARY_DIR} 0)
 
-  add_custom_command(OUTPUT python/python/opm/${python_lib_target}
+  add_custom_command(OUTPUT python/opm/${python_lib_target}
                      DEPENDS ${PYTHON_CXX_DEPENDS}
                      DEPENDS copy_python
                      COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_BINARY_DIR}/python/setup.py
                                                       build
                                                       build_ext
-                                                      --build-lib=${PROJECT_BINARY_DIR}/python/python/opm
+                                                      --build-lib=${PROJECT_BINARY_DIR}/python
                                                       --library-dirs=${_setup_lib_dirs}
                                                       ${_rpath_arg}
                                                       --include-dirs=${_setup_include_dirs}
                      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python
-                     COMMENT "Building python bindings")
+                     COMMENT "Building python bindings at python/opm/${python_lib_target}")
-  add_custom_target(opmcommon_python ALL DEPENDS python/python/opm/${python_lib_target})
+  add_custom_target(opmcommon_python ALL DEPENDS python/opm/${python_lib_target})
   add_dependencies(opmcommon_python opmcommon)
 
   # The install target is based on manually copying the python file tree to the
@@ -318,7 +324,7 @@ if (OPM_ENABLE_PYTHON)
   # setup.py install manually - optionally with the generated script
   # setup-install.sh - and completely bypass cmake in the installation phase.
   if (OPM_INSTALL_PYTHON)
-    install( CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_BINARY_DIR}/python/install.py ${PROJECT_BINARY_DIR}/python/python/opm ${DEST_PREFIX}${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_PREFIX} 1)")
+    install( CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_BINARY_DIR}/python/install.py ${PROJECT_BINARY_DIR}/python/opm ${DEST_PREFIX}${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_PREFIX} 1)")
   endif()
 
   # Observe that if the opmcommon library has been built as a shared library the
@@ -326,11 +332,11 @@ if (OPM_ENABLE_PYTHON)
   # testing.
   add_test(NAME python_tests
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/python
-           COMMAND ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/lib ${PYTHON_EXECUTABLE} setup.py build_ext --dry-run --build-lib ${PROJECT_BINARY_DIR}/python/python/opm test
+           COMMAND ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/lib ${PYTHON_EXECUTABLE} setup.py build_ext --dry-run --build-lib ${PROJECT_BINARY_DIR}/python test
            )
 
   set_target_properties(opmcommon PROPERTIES POSITION_INDEPENDENT_CODE ON)
-  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${PROJECT_BINARY_DIR}/python/python)
+  set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${PROJECT_BINARY_DIR}/python)
 
   # -------------------------------------------------------------------------
   # Let cmake configure some small shell scripts which can be used to simplify
@@ -340,6 +346,11 @@ if (OPM_ENABLE_PYTHON)
         DESTINATION ${PROJECT_BINARY_DIR}
         FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE )
 
+  configure_file(python/setup-package.sh.in tmp/setup-package.sh)
+  file( COPY ${PROJECT_BINARY_DIR}/tmp/setup-package.sh
+        DESTINATION ${PROJECT_BINARY_DIR}
+        FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE )
+
   configure_file(python/setup-test.sh.in tmp/setup-test.sh)
   file( COPY ${PROJECT_BINARY_DIR}/tmp/setup-test.sh
         DESTINATION ${PROJECT_BINARY_DIR}

python/Dockerfile

@@ -0,0 +1,12 @@
+# Dockerfile to generate PyPI packages. Needs to be run from the opm-common root folder
+# Example use:
+# sudo docker build -t manylinux2014_opm:built . -f python/Dockerfile
+
+FROM quay.io/pypa/manylinux2014_x86_64
+ARG version_tag=""
+WORKDIR /tmp/opm-common
+RUN echo "Using package version tag: $version_tag"
+ADD . .
+RUN /bin/bash /tmp/opm-common/python/generate-pypi-package.sh $version_tag
+# docker run -e PLAT=manylinux2014_x86_64 -it lindkvis/manylinux2014_opm:latest
+

python/Dockerfile.create

@@ -0,0 +1,8 @@
+# Docker file to generate a Docker image capable of building PyPI packages
+# Example use:
+# Use sudo docker build -t lindkvis/manylinux2014_opm:latest -f Dockerfile.create .
+
+FROM quay.io/pypa/manylinux2014_x86_64
+WORKDIR /tmp
+COPY setup-docker-image.sh .
+RUN /bin/bash /tmp/setup-docker-image.sh

python/README.md

@@ -0,0 +1,2 @@
+Python bindings for the OPM-common module of the Open Porous Media project.
+

python/generate-pypi-package.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Run in docker container
+# docker run -e PLAT=manylinux2014_x86_64 -it quay.io/pypa/manylinux2014_x86_64
+
+VERSION_TAG=${1:-""}
+
+#export PYTHON27=/usr/bin/python2.7
+export PYTHON35=/opt/python/cp35-cp35m/bin/python
+export PYTHON36=/opt/python/cp36-cp36m/bin/python
+export PYTHON37=/opt/python/cp37-cp37m/bin/python
+export PYTHON38=/opt/python/cp38-cp38/bin/python
+
+/bin/bash /tmp/opm-common/python/setup-docker-image.sh
+
+cd /tmp/opm-common
+
+# Delete the folder if it already exists
+if [ -d build ]; then
+  rm -rf build
+fi
+
+mkdir build && cd build
+
+cmake3 -DPYTHON_EXECUTABLE=${PYTHON35} -DBOOST_INCLUDEDIR=/usr/include/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 \
+-DOPM_ENABLE_PYTHON=ON -DOPM_PYTHON_PACKAGE_VERSION_TAG=${VERSION_TAG} ..
+
+# make step is necessary until the generated ParserKeywords/*.hpp are generated in the Python step
+make -j2
+
+./setup-package.sh
+${PYTHON35} -m auditwheel repair python/dist/*cp35*.whl
+
+# Run setup-package.sh four times (Python 3.5, 3.6, 3.7 and 3.8)
+cmake3 -DPYTHON_EXECUTABLE=${PYTHON36} -DBOOST_INCLUDEDIR=/usr/include/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 \
+-DOPM_ENABLE_PYTHON=ON -DOPM_PYTHON_PACKAGE_VERSION_TAG=${VERSION_TAG} ..
+./setup-package.sh
+${PYTHON36} -m auditwheel repair python/dist/*cp36*.whl
+
+cmake3 -DPYTHON_EXECUTABLE=${PYTHON37} -DBOOST_INCLUDEDIR=/usr/include/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 \
+-DOPM_ENABLE_PYTHON=ON -DOPM_PYTHON_PACKAGE_VERSION_TAG=${VERSION_TAG} ..
+./setup-package.sh
+${PYTHON37} -m auditwheel repair python/dist/*cp37*.whl
+
+cmake3 -DPYTHON_EXECUTABLE=${PYTHON38} -DBOOST_INCLUDEDIR=/usr/include/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 \
+-DOPM_ENABLE_PYTHON=ON -DOPM_PYTHON_PACKAGE_VERSION_TAG=${VERSION_TAG} ..
+./setup-package.sh
+${PYTHON38} -m auditwheel repair python/dist/*cp38*.whl
+
+#cmake3 -DPYTHON_EXECUTABLE=${PYTHON27} -DBOOST_INCLUDEDIR=/usr/include/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 \
+#-DOPM_ENABLE_PYTHON=ON -DOPM_ENABLE_DYNAMIC_BOOST=OFF -DOPM_ENABLE_DYNAMIC_PYTHON_LINKING=OFF ..
+#./setup-package.sh
+#${PYTHON27} -m auditwheel repair python/dist/*cp27*.whl
+
+
+# Example of upload
+# /usr/bin/python3 -m twine upload --repository testpypi wheelhouse/*

python/install.py

@@ -8,7 +8,7 @@ src_root = sys.argv[1]
 target_prefix = sys.argv[2]
 install = int(sys.argv[3])
 target_destdir = os.environ.get("DESTDIR", "")
-if not target_destdir is "":
+if target_destdir != "":
     target_prefix = target_destdir + target_prefix
 
 if not os.path.isdir(src_root):

python/pypi-howto.txt

@@ -0,0 +1,20 @@
+#  How to build pypi-packages:
+
+* For initial python package version. Run in the opm-common root folder:
+
+sudo docker build -t manylinux2014_opm:built . -f python/Dockerfile
+
+* This will create a docker image with a version looking like '2020.10' with package 
+  wheels ready for upload in /tmp/opm-common/build/wheelhouse
+* For subsequent python package versions, add a version_tag build argument including a dot '.':
+
+sudo docker build -t manylinux2014_opm:built --build-arg version_tag=".1" . -f python/Dockerfile
+
+* This will create a python package version looking like '2020.10.1'.
+* Once the image is finished, run it:
+
+sudo docker run -it manylinux2014_opm:built
+
+* The package wheels can then be uploaded to test-pypi with:
+
+/opt/python/cp38-cp38/bin/python -m twine upload -u __token__ -p TOKEN_VALUE --repository testpypi /tmp/opm-common/build/wheelhouse/*.whl

python/setup-build.sh.in

@@ -10,11 +10,11 @@
 
 cp -r @PROJECT_SOURCE_DIR@/python @PROJECT_BINARY_DIR@
 cd @PROJECT_BINARY_DIR@/python
-rm -f python/opm/libopmcommon_python*
+rm -f opm/libopmcommon_python*
 
 export CC=@CMAKE_CXX_COMPILER@
 
-@PYTHON_EXECUTABLE@ setup.py build build_ext --build-lib=python/opm \
+@PYTHON_EXECUTABLE@ setup.py build build_ext \
 --library-dirs=@_setup_lib_dirs@ \
 @_rpath_arg@ \
 --include-dirs=@_setup_include_dirs@

python/setup-docker-image.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Script to be run on a manylinux2014 docker image to complete it for OPM usage.
+# i.e. docker run -i -t quay.io/pypa/manylinux2014_x86_64 < setup-docker-image.sh
+
+# A ready made Docker image is available at Dockerhub:
+# docker run -i -t lindkvis/manylinux2014_opm:latest
+
+yum-config-manager --add-repo \
+https://www.opm-project.org/package/opm.repo
+yum install -y cmake3 ccache boost169-devel boost169-static
+yum install -y blas-devel suitesparse-devel trilinos-openmpi-devel
+
+#${PYTHON27} -m pip install pip --upgrade
+#${PYTHON27} -m pip install wheel setuptools twine pytest-runner auditwheel
+${PYTHON35} -m pip install pip --upgrade
+${PYTHON35} -m pip install wheel setuptools twine pytest-runner auditwheel
+${PYTHON36} -m pip install pip --upgrade
+${PYTHON36} -m pip install wheel setuptools twine pytest-runner auditwheel
+${PYTHON37} -m pip install pip --upgrade
+${PYTHON37} -m pip install wheel setuptools twine pytest-runner auditwheel
+${PYTHON38} -m pip install pip --upgrade
+${PYTHON38} -m pip install wheel setuptools twine pytest-runner auditwheel
+

python/setup-package.sh.in

@@ -0,0 +1,20 @@
+#!/bin/bash
+# This cmake template file can be used to create a small shell script which can
+# be used to generate a 
+# script setup-build.sh from this templated file - that script can be used to
+# run the python setup.py process without going through cmake.
+#
+# The script in question is purely a convenience for Python development, it is
+# fully optional to use it, and it is not used by the main cmake based build
+# system.
+
+cp -r @PROJECT_SOURCE_DIR@/python @PROJECT_BINARY_DIR@
+cd @PROJECT_BINARY_DIR@/python
+rm -rf opm/libopmcommon_python*
+
+export CC=@CMAKE_CXX_COMPILER@
+
+@PYTHON_EXECUTABLE@ setup.py sdist bdist_wheel build_ext \
+--library-dirs=@_setup_lib_dirs@ \
+@_rpath_arg@ \
+--include-dirs=@_setup_include_dirs@

python/setup-test.sh.in

@@ -10,8 +10,8 @@
 
 cp -r @PROJECT_SOURCE_DIR@/python @PROJECT_BINARY_DIR@
 cd @PROJECT_BINARY_DIR@/python
-export PYTHONPATH=@PROJECT_BINARY_DIR@/python/python:$PYTHONPATH
+export PYTHONPATH=@PROJECT_BINARY_DIR@/python:$PYTHONPATH
 export LD_LIBRARY_PATH=@PROJECT_BINARY_DIR@/lib:@_setup_lib_dirs@:$LD_LIBRARY_PATH
 
 
-@PYTHON_EXECUTABLE@ setup.py build_ext --dry-run --build-lib=python/opm test
+@PYTHON_EXECUTABLE@ setup.py build_ext --dry-run

python/setup.py.in

@@ -40,7 +40,7 @@ if 'build' in sys.argv:
 
 ext_modules = [
     Extension(
-        'libopmcommon_python',
+        'opm.libopmcommon_python',
         [
                 'cxx/unit_system.cpp',
                 'cxx/connection.cpp',
@@ -67,13 +67,22 @@ ext_modules = [
         undef_macros=["NDEBUG"],
         include_dirs=["pybind11/include"],
         extra_compile_args=['-std=c++17', '-fopenmp'],
-        extra_link_args=['-fopenmp']
+        extra_link_args=['-fopenmp', @opm-common_PYTHON_LINKAGE@]
     )
 ]
 
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
 setup(
-    name='Opm',
+    name='opm',
-    package_dir = {'': 'python'},
+    version = '@opm-common_VERSION@' + '@opm-common_PYTHON_PACKAGE_VERSION@',
+    url='http://www.opm-project.org',
+    author='The Open Porous Media Project',
+    author_email='opmuser@gmail.com',
+    description='OPM-Common Python bindings',
+    long_description=long_description,
+    long_description_content_type="text/markdown",
     packages=[
                 'opm',
                 'opm.io',
@@ -85,10 +94,14 @@ setup(
                 'opm.tools'
             ],
     ext_modules=ext_modules,
-    package_data={'opm': ['libopmcommon_python{}'.format(suffix)]},
     include_package_data=True,
     license='Open Source',
     zip_safe=False,
     test_suite='tests',
     setup_requires=["pytest-runner", 'setuptools_scm'],
+    python_requires='>=3.5',
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    ],
 )