Merge pull request #1075 from joakim-hove/python-install

Add Python copy/install hack

CMakeLists.txt

@@ -9,6 +9,7 @@ option(ENABLE_ECL_INPUT "Enable eclipse input support?" ON)
 option(ENABLE_ECL_OUTPUT "Enable eclipse output support?" ON)
 option(ENABLE_MOCKSIM "Build the mock simulator for io testing" ON)
 option(OPM_ENABLE_PYTHON "Enable python bindings?" OFF)
+option(OPM_INSTALL_PYTHON "Enable python bindings?" OFF)
 option(OPM_ENABLE_EMBEDDED_PYTHON "Enable python bindings?" OFF)
 
 
@@ -220,10 +221,10 @@ if (OPM_ENABLE_PYTHON)
   find_package(PythonInterp REQUIRED)
   include(FindPythonInterp)
 
-  make_directory(${CMAKE_BINARY_DIR}/python)
   set(PYTHON_PACKAGE_PATH "site-packages")
   set(PYTHON_INSTALL_PREFIX "lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/${PYTHON_PACKAGE_PATH}" CACHE STRING "Subdirectory to install Python modules in")
 
+  make_directory(${CMAKE_BINARY_DIR}/python)
   get_target_property(_ecl_include_dirs ecl INTERFACE_INCLUDE_DIRECTORIES)
   get_target_property(_opmcommon_include_dirs opmcommon INCLUDE_DIRECTORIES)
   list(APPEND _opmcommon_include_dirs ${_ecl_include_dirs})
@@ -268,8 +269,20 @@ if (OPM_ENABLE_PYTHON)
   add_custom_target(opmcommon_python ALL DEPENDS python/python/opm/libopmcommon_python.so)
   add_dependencies(opmcommon_python opmcommon)
 
-  install( CODE "execute_process(COMMAND mkdir -p ${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_PREFIX}  )" )
-  install( CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} python/setup.py build_ext --dry-run install --prefix=${CMAKE_INSTALL_PREFIX}  )" )
+  # The install target is based on manually copying the python file tree to the
+  # installation area with a small installation script 'install.py'. Would have
+  # preferred to use standard setup.py install, but the setup.py based solution
+  # refuses to install to a location which the current python executable can not
+  # load from, and the use of eggs in the setup.py based installation makes
+  # debugging quite difficult.
+  #
+  # Since the installation of Python code is nonstandard it is protected by an
+  # extra cmake switch, OPM_INSTALL_PYTHON. If you prefer you can still invoke
+  # setup.py install manually - optionally with the generated script
+  # setup-install.sh - and completely bypass cmake in the installation phase.
+  if (OPM_INSTALL_PYTHON)
+    install( CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_BINARY_DIR}/python/install.py ${CMAKE_BINARY_DIR}/python/python/opm ${CMAKE_INSTALL_PREFIX}/${PYTHON_INSTALL_PREFIX})")
+  endif()
 
   add_test(NAME python_tests
             WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/python
@@ -281,7 +294,7 @@ if (OPM_ENABLE_PYTHON)
 
   # -------------------------------------------------------------------------
   # Let cmake configure some small shell scripts which can be used to simplify
-  # building and testing of the Python extensions.
+  # building, testing and installation of the Python extensions.
   configure_file(python/setup-build.sh.in tmp/setup-build.sh)
   file( COPY ${PROJECT_BINARY_DIR}/tmp/setup-build.sh
         DESTINATION ${PROJECT_BINARY_DIR}
@@ -292,6 +305,11 @@ if (OPM_ENABLE_PYTHON)
         DESTINATION ${PROJECT_BINARY_DIR}
         FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE )
 
+  configure_file(python/setup-install.sh.in tmp/setup-install.sh)
+  file( COPY ${PROJECT_BINARY_DIR}/tmp/setup-install.sh
+        DESTINATION ${PROJECT_BINARY_DIR}
+        FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE )
+
   configure_file(python/enable-python.sh.in enable-python.sh)
 
   # -------------------------------------------------------------------------

jenkins/build-opm-module.sh

@@ -4,7 +4,7 @@ declare -A configurations
 
 declare -A EXTRA_MODULE_FLAGS
 EXTRA_MODULE_FLAGS[opm-simulators]="-DBUILD_EBOS_EXTENSIONS=ON -DBUILD_EBOS_DEBUG_EXTENSIONS=ON -DBUILD_FLOW_VARIANTS=ON"
-EXTRA_MODULE_FLAGS[opm-common]="-DOPM_ENABLE_PYTHON=ON -DOPM_ENABLE_EMBEDDED_PYTHON=ON"
+EXTRA_MODULE_FLAGS[opm-common]="-DOPM_ENABLE_PYTHON=ON -DOPM_ENABLE_EMBEDDED_PYTHON=ON -DOPM_INSTALL_PYTHON=ON"
 EXTRA_MODULE_FLAGS[libecl]="-DCMAKE_POSITION_INDEPENDENT_CODE=1"
 
 # Parse revisions from trigger comment and setup arrays

python/install.py

@@ -0,0 +1,31 @@
+import sys
+import os
+import shutil
+import compileall
+
+src_root = sys.argv[1]
+target_prefix = sys.argv[2]
+
+
+if not os.path.isdir(src_root):
+    sys.exit("No such directory: {}".format(src_root))
+
+path_offset = len(os.path.dirname(src_root))
+for path,_ ,fnames in os.walk(src_root):
+    target_path = os.path.join(target_prefix, path[path_offset+1:])
+    if not os.path.isdir(target_path):
+        print("-- Installing: {}".format(target_path))
+        os.makedirs(target_path)
+
+    for f in fnames:
+        _, ext = os.path.splitext(f)
+        if ext == ".pyc":
+            continue
+
+        src_file = os.path.join(path, f)
+        target_file = os.path.join(target_path, f)
+        shutil.copy(src_file, target_file)
+        print("-- Installing: {}".format(target_file))
+
+target_root = os.path.join(target_prefix, os.path.basename(src_root))
+compileall.compile_dir(target_root)

python/setup-install.sh.in

@@ -0,0 +1,17 @@
+#!/bin/bash
+# This cmake template file can be used to create a small shell script which can
+# be used to simplify the Python development process. The cmake step will create
+# script setup-install.sh from this templated file - that script can be used to
+# run the installation through the setup.py macinery without going through
+# cmake.
+#
+# The script in question is purely a convenience for Python development, it is
+# fully optional to use it, and it is not used by the main cmake based build
+# system.
+
+cd @PROJECT_BINARY_DIR@/python
+export PYTHONPPATH=@CMAKE_INSTALL_PREFIX@/@PYTHON_PREFIX
+export LD_LIBRARY_PATH=@PROJECT_BINARY_DIR@/lib:@_setup_lib_dirs@:$LD_LIBRARY_PATH
+
+
+@PYTHON_EXECUTABLE@ setup.py build_ext --dry-run install --prefix=@CMAKE_INSTALL_PREFIX@

python/setup.py

@@ -8,28 +8,6 @@ import setuptools
 import glob
 import os
 import subprocess
-import argparse
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument("build")
-parser.add_argument("build_ext")
-parser.add_argument("--build-lib")
-parser.add_argument("--prefix")
-parser.add_argument("--library-dirs")
-parser.add_argument("--include-dirs")
-parser.add_argument("--dry-run", action='store_true')
-args = parser.parse_args()
-
-#Circumnventing setuptools' PYTHONPATH check.
-#This is necessary during install
-if args.prefix:
-   python_version = 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)
-   pkg_path_root = os.path.join(args.prefix, 'lib', python_version, 'site-packages')
-   if 'PYTHONPATH' in os.environ:
-       os.environ['PYTHONPATH'] += ':' + pkg_path_root
-   else:
-       os.environ['PYTHONPATH'] = pkg_path_root
 
 setupdir = os.path.dirname(__file__)
 if setupdir != '':