Created prelimary cmake build

2013-12-05 11:43:40 -05:00
1 parent 9dd97d7f0f
commit 0430704252
@@ -0,0 +1,111 @@
+# Set some CMake properties    
+CMAKE_MINIMUM_REQUIRED(VERSION 2.6)
+CMAKE_POLICY(SET CMP0011 OLD)
+CMAKE_POLICY(SET CMP0014 OLD)
+
+
+MESSAGE("====================")
+MESSAGE("Configuring LBPM-WIA")
+MESSAGE("====================")
+
+
+# Set the project name
+PROJECT( LBPM-WIA )
+
+
+# Check if we are only compiling docs
+INCLUDE( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake )
+INCLUDE( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/libraries.cmake )
+CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 )
+
+
+# Set testing paramaters
+SET( DROP_METHOD "http" )
+SET( DROP_SITE "oblivion.engr.colostate.edu" )
+SET( DROP_LOCATION "/CDash/submit.php?project=AMR-MHD" )
+SET( TRIGGER_SITE "" )
+SET( DROP_SITE_CDASH TRUE )
+ENABLE_TESTING()
+INCLUDE( CTest )
+
+
+# Set some common paths
+SET( LBPM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} )
+SET( LBPM_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR} )
+IF( PREFIX )
+    SET( LBPM_INSTALL_DIR ${PREFIX} )
+ELSEIF( NOT LBPM_INSTALL_DIR )
+    SET( LBPM_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR} )
+ENDIF()
+SET( CMAKE_MODULE_PATH ${LBPM_SOURCE_DIR} ${LBPM_SOURCE_DIR}/cmake )
+
+
+# Create custom targets for build-test, check, and distclean
+ADD_CUSTOM_TARGET( doc )
+ADD_CUSTOM_TARGET( latex_docs )
+ADD_CUSTOM_TARGET( build-test )
+ADD_CUSTOM_TARGET( check COMMAND  make test  )
+ADD_DISTCLEAN()
+
+
+# Check the compile mode and compile flags
+IF ( NOT ONLY_BUILD_DOCS )
+    CONFIGURE_SYSTEM()
+ENDIF()
+
+
+# Add some directories to include
+INCLUDE_DIRECTORIES( ${LBPM_INSTALL_DIR}/include )
+
+
+# Set doxygen info
+CHECK_ENABLE_FLAG( USE_DOXYGEN 1 )
+CHECK_ENABLE_FLAG( USE_LATEX 1 )
+FILE( MAKE_DIRECTORY "${LBPM_INSTALL_DIR}/doc" )
+IF ( USE_DOXYGEN )
+    SET( DOXYFILE_IN ${LBPM_SOURCE_DIR}/doxygen/Doxyfile.in)
+    SET( DOXY_HEADER_FILE ${LBPM_SOURCE_DIR}/doxygen/html/header.html )
+    SET( DOXY_FOOTER_FILE ${LBPM_SOURCE_DIR}/doxygen/html/footer.html )
+    SET( DOXYFILE_OUTPUT_DIR ${LBPM_BUILD_DIR}/doc/ )
+    SET( DOXYFILE_SRC_HTML_DIR ${LBPM_SOURCE_DIR}/doxygen/html )
+    SET( DOXYFILE_SOURCE_DIR ${LBPM_SOURCE_DIR} )
+    SET( REL_PACKAGE_HTML "" )
+    SET( DOXYGEN_MACROS "" )
+    MESSAGE("DOXYGEN_MACROS = ${DOXYGEN_MACROS}")
+    INCLUDE( ${LBPM_SOURCE_DIR}/cmake/UseDoxygen.cmake )
+    SET(DOXYFILE_LATEX "YES")
+    IF ( DOXYGEN_FOUND )
+        ADD_DEPENDENCIES( doxygen latex_docs )
+        ADD_DEPENDENCIES( doc latex_docs doxygen )
+    ELSE()
+        SET( USE_DOXYGEN 0 )
+    ENDIF()
+ENDIF()
+
+
+# Configure external packages
+IF ( NOT ONLY_BUILD_DOCS )
+    CONFIGURE_MPI()     # MPI must be before other libraries
+    CONFIGURE_LBPM()
+    CONFIGURE_LINE_COVERAGE()
+ENDIF()
+
+
+# Configure internal libraries (order matters)
+SET( LBPM_LIBS  lbpm-wia )
+
+
+# Add the src directories
+IF ( NOT ONLY_BUILD_DOCS )
+    BEGIN_PACKAGE_CONFIG( lbpm-wia )
+    ADD_PACKAGE_SUBDIRECTORY( include )
+    #ADD_PACKAGE_SUBDIRECTORY( cpu )
+    ADD_PACKAGE_SUBDIRECTORY( gpu )
+    INSTALL_LBPM_TARGET( lbpm-wia )
+    ADD_SUBDIRECTORY( tests )
+    #ADD_SUBDIRECTORY( cpu/exe )
+    ADD_SUBDIRECTORY( gpu/exe )
+    INSTALL( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/example DESTINATION ${LBPM_INSTALL_DIR} )
+ENDIF()
+
+
@@ -1,63 +0,0 @@
-CUDA_FLAGS=-arch sm_35
-
-all:bin/Color-WIA bin/Color-WIA-CBUB
-
-bin/Color-WIA:gpu/lb2_Color_wia_mpi.o lib/libcuColor.a lib/libcuD3Q19.a lib/libcuD3Q7.a lib/libcuExtras.a
-	mkdir -p bin
-	CC -O3 -o bin/Color-WIA gpu/lb2_Color_wia_mpi.o -lcuColor -lcuD3Q19 -lcuD3Q7 -lcuExtras -Llib -Iinclude
-
-
-bin/Color-WIA-CBUB:gpu/lb2_Color_wia_mpi_cbub.o lib/libcuColor.a lib/libcuD3Q19.a lib/libcuD3Q7.a lib/libcuExtras.a
-	mkdir -p bin
-	CC -O3 -D CBUB -o bin/Color-WIA-CBUB gpu/lb2_Color_wia_mpi_cbub.o -lcuColor -lcuD3Q19 -lcuD3Q7 -lcuExtras -Llib -Iinclude
-
-
-bin/ColorLBM:gpu/lb2_Color_mpi.cpp lib/libcuColor.a lib/libcuD3Q19.a lib/libcuD3Q7.a lib/libcuExtras.a
-	mkdir -p bin
-	CC -O3 -o bin/ColorLBM gpu/lb2_Color_mpi.cpp -lcuColor -lcuD3Q19 -lcuD3Q7 -lcuExtras -Llib
-
-#bin/gpuMRT:gpu/lb1_MRT.cu lib/libcuMRT.a lib/libcuD3Q19.a
-#	mkdir -p bin
-#	nvcc -O3 -o bin/gpuMRT $(CUDA_FLAGS) gpu/lb1_MRT.cu -lcuMRT -lcuD3Q19 -Llib
-
-#bin/gpuColor:gpu/lb2_Color.cu lib/libcuColor.a lib/libcuD3Q19.a
-#	mkdir -p bin
-#	nvcc -o bin/gpuColor $(CUDA_FLAGS) gpu/lb2_Color.cu -lcuColor -lcuD3Q19 -Llib
-
-gpu/lb2_Color_wia_mpi.o:gpu/lb2_Color_wia_mpi.cpp
-	CC -c -o gpu/lb2_Color_wia_mpi.o gpu/lb2_Color_wia_mpi.cpp -Iinclude
- 
-
-gpu/lb2_Color_wia_mpi_cbub.o:gpu/lb2_Color_wia_mpi.cpp
-	CC -c -DCBUB -o gpu/lb2_Color_wia_mpi_cbub.o gpu/lb2_Color_wia_mpi.cpp -Iinclude
-
-#pmmc/pmmc.o:pmmc/pmmc.cpp
-#	CC -c -o pmmc/pmmc.o pmmc/pmmc.cpp -Iinclude
-
-#include/Array.o:pmmc/Array.cpp
-#	CC -c -o include/Array.o include/Array.cpp -Iinclude
-
-lib/libcuExtras.a: gpu/CudaExtras.cu
-	mkdir -p lib
-	nvcc -lib $(CUDA_FLAGS) gpu/CudaExtras.cu -o lib/libcuExtras.a
-
-#lib/libcuMRT.a: gpu/MRT.cu
-#	mkdir -p lib
-#	nvcc -lib $(CUDA_FLAGS) gpu/MRT.cu -o lib/libcuMRT.a
-
-lib/libcuD3Q7.a: gpu/D3Q7.cu
-	mkdir -p lib
-	nvcc -lib $(CUDA_FLAGS) gpu/D3Q7.cu -o lib/libcuD3Q7.a
-
-lib/libcuD3Q19.a: gpu/D3Q19.cu
-	mkdir -p lib
-	nvcc -lib $(CUDA_FLAGS) gpu/D3Q19.cu -o lib/libcuD3Q19.a
-
-lib/libcuColor.a: gpu/Color.cu
-	mkdir -p lib
-	nvcc -lib $(CUDA_FLAGS) gpu/Color.cu -o lib/libcuColor.a
-
-clean:
-	rm gpu/*.o
-	rm bin/*
-	rm lib/*
@@ -0,0 +1,362 @@
+# - Message Passing Interface (MPI) module.
+#
+# The Message Passing Interface (MPI) is a library used to write
+# high-performance parallel applications that use message passing, and
+# is typically deployed on a cluster. MPI is a standard interface
+# (defined by the MPI forum) for which many implementations are
+# available. All of these implementations have somewhat different
+# compilation approaches (different include paths, libraries to link
+# against, etc.), and this module tries to smooth out those differences.
+#
+# This module will set the following variables:
+#   MPI_FOUND                  TRUE if we have found MPI
+#   MPI_COMPILE_FLAGS          Compilation flags for MPI programs
+#   MPI_INCLUDE_PATH           Include path(s) for MPI header
+#   MPI_LINK_FLAGS             Linking flags for MPI programs
+#   MPI_LIBRARY                First MPI library to link against (cached)
+#   MPI_EXTRA_LIBRARY          Extra MPI libraries to link against (cached)
+#   MPI_LIBRARIES              All libraries to link MPI programs against
+#   MPIEXEC                    Executable for running MPI programs
+#   MPIEXEC_NUMPROC_FLAG       Flag to pass to MPIEXEC before giving it the
+#                              number of processors to run on
+#   MPIEXEC_PREFLAGS           Flags to pass to MPIEXEC directly before the
+#                              executable to run.
+#   MPIEXEC_POSTFLAGS          Flags to pass to MPIEXEC after all other flags.
+#
+# This module will attempt to auto-detect these settings, first by
+# looking for a MPI compiler, which many MPI implementations provide
+# as a pass-through to the native compiler to simplify the compilation
+# of MPI programs. The MPI compiler is stored in the cache variable
+# MPI_COMPILER, and will attempt to look for commonly-named drivers
+# mpic++, mpicxx, mpiCC, or mpicc. If the compiler driver is found and
+# recognized, it will be used to set all of the module variables. To
+# skip this auto-detection, set MPI_LIBRARY and MPI_INCLUDE_PATH in
+# the CMake cache.
+#
+# If no compiler driver is found or the compiler driver is not
+# recognized, this module will then search for common include paths
+# and library names to try to detect MPI.
+#
+# If CMake initially finds a different MPI than was intended, and you
+# want to use the MPI compiler auto-detection for a different MPI
+# implementation, set MPI_COMPILER to the MPI compiler driver you want
+# to use (e.g., mpicxx) and then set MPI_LIBRARY to the string
+# MPI_LIBRARY-NOTFOUND. When you re-configure, auto-detection of MPI
+# will run again with the newly-specified MPI_COMPILER.
+#
+# When using MPIEXEC to execute MPI applications, you should typically
+# use all of the MPIEXEC flags as follows:
+#   ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS ${MPIEXEC_PREFLAGS} EXECUTABLE
+#     ${MPIEXEC_POSTFLAGS} ARGS
+# where PROCS is the number of processors on which to execute the program,
+# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the
+# MPI program.
+
+#=============================================================================
+# Copyright 2001-2009 Kitware, Inc.
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+# This module is maintained by David Partyka <dave.partyka@kitware.com>.
+
+# A set of directories to search through in addition to the standard system paths
+# that find_program will search through.
+# Microsoft HPC SDK is automatically added to the system path
+# Argonne National Labs MPICH2 sets a registry key that we can use.
+
+set(_MPI_PACKAGE_DIR
+  mpi
+  mpich
+  openmpi
+  lib/mpi
+  lib/mpich
+  lib/openmpi
+  "MPICH/SDK"
+  "Microsoft Compute Cluster Pack"
+  "Microsoft HPC Pack 2008 R2"
+  )
+
+set(_MPI_PREFIX_PATH)
+if(WIN32)
+  list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..")
+  list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]")
+endif()
+
+foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH})
+  foreach(MpiPackageDir ${_MPI_PREFIX_PATH})
+    if(EXISTS ${SystemPrefixDir}/${MpiPackageDir})
+      list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}")
+    endif()
+  endforeach(MpiPackageDir)
+endforeach(SystemPrefixDir)
+
+# Most mpi distros have some form of mpiexec which gives us something we can reliably look for.
+find_program(MPIEXEC
+  NAMES mpiexec mpirun lamexec
+  PATHS ${_MPI_PREFIX_PATH}
+  PATH_SUFFIXES bin
+  DOC "Executable for running MPI programs."
+  )
+
+# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin).
+# This gives us a fairly reliable base directory to search for /bin /lib and /include from.
+get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH)
+get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH)
+
+# If there is an mpi compiler find it and interogate (farther below) it for the include
+# and lib dirs otherwise we will continue to search from ${_MPI_BASE_DIR}.
+find_program(MPI_COMPILER
+  NAMES mpic++ mpicxx mpiCC mpicc
+  HINTS "${_MPI_BASE_DIR}"
+  PATH_SUFFIXES bin
+  DOC "MPI compiler. Used only to detect MPI compilation flags.")
+mark_as_advanced(MPI_COMPILER)
+
+set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.")
+set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.")
+set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.")
+set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.")
+mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS
+  MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS)
+
+if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+  # Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
+  # the cache, and we don't want to override those settings.
+elseif (MPI_COMPILER)
+  # Check whether the -showme:compile option works. This indicates
+  # that we have either Open MPI or a newer version of LAM-MPI, and
+  # implies that -showme:link will also work.
+  # Note that Windows distros do not have an mpi compiler to interogate.
+  exec_program(${MPI_COMPILER}
+    ARGS -showme:compile
+    OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
+    RETURN_VALUE MPI_COMPILER_RETURN)
+
+  if (MPI_COMPILER_RETURN EQUAL 0)
+    # If we appear to have -showme:compile, then we should also have
+    # -showme:link. Try it.
+    exec_program(${MPI_COMPILER}
+      ARGS -showme:link
+      OUTPUT_VARIABLE MPI_LINK_CMDLINE
+      RETURN_VALUE MPI_COMPILER_RETURN)
+
+    # Note that we probably have -showme:incdirs and -showme:libdirs
+    # as well.
+    set(MPI_COMPILER_MAY_HAVE_INCLIBDIRS TRUE)
+  endif (MPI_COMPILER_RETURN EQUAL 0)
+
+  if (MPI_COMPILER_RETURN EQUAL 0)
+    # Do nothing: we have our command lines now
+  else (MPI_COMPILER_RETURN EQUAL 0)
+    # Older versions of LAM-MPI have "-showme". Try it.
+    exec_program(${MPI_COMPILER}
+      ARGS -showme
+      OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
+      RETURN_VALUE MPI_COMPILER_RETURN)
+  endif (MPI_COMPILER_RETURN EQUAL 0)
+
+  if (MPI_COMPILER_RETURN EQUAL 0)
+    # Do nothing: we have our command lines now
+  else (MPI_COMPILER_RETURN EQUAL 0)
+    # MPICH uses "-show". Try it.
+    exec_program(${MPI_COMPILER}
+      ARGS -show
+      OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
+      RETURN_VALUE MPI_COMPILER_RETURN)
+  endif (MPI_COMPILER_RETURN EQUAL 0)
+
+  if (MPI_COMPILER_RETURN EQUAL 0)
+    # We have our command lines, but we might need to copy
+    # MPI_COMPILE_CMDLINE into MPI_LINK_CMDLINE, if the underlying
+    if (NOT MPI_LINK_CMDLINE)
+      SET(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE})
+    endif (NOT MPI_LINK_CMDLINE)
+  else (MPI_COMPILER_RETURN EQUAL 0)
+    message(STATUS "Unable to determine MPI from MPI driver ${MPI_COMPILER}")
+  endif (MPI_COMPILER_RETURN EQUAL 0)
+endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+
+if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+  # Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
+  # the cache, and we don't want to override those settings.
+elseif (MPI_COMPILE_CMDLINE)
+  # Extract compile flags from the compile command line.
+  string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}")
+  set(MPI_COMPILE_FLAGS_WORK)
+  foreach(FLAG ${MPI_ALL_COMPILE_FLAGS})
+    if (MPI_COMPILE_FLAGS_WORK)
+      set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}")
+    else(MPI_COMPILE_FLAGS_WORK)
+      set(MPI_COMPILE_FLAGS_WORK ${FLAG})
+    endif(MPI_COMPILE_FLAGS_WORK)
+  endforeach(FLAG)
+
+  # Extract include paths from compile command line
+  string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}")
+  set(MPI_INCLUDE_PATH_WORK)
+  foreach(IPATH ${MPI_ALL_INCLUDE_PATHS})
+    string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH})
+    string(REGEX REPLACE "//" "/" IPATH ${IPATH})
+    list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH})
+  endforeach(IPATH)
+
+  if (NOT MPI_INCLUDE_PATH_WORK)
+    if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
+      # The compile command line didn't have any include paths on it,
+      # but we may have -showme:incdirs. Use it.
+      exec_program(${MPI_COMPILER}
+        ARGS -showme:incdirs
+        OUTPUT_VARIABLE MPI_INCLUDE_PATH_WORK
+        RETURN_VALUE MPI_COMPILER_RETURN)
+      separate_arguments(MPI_INCLUDE_PATH_WORK)
+    endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
+  endif (NOT MPI_INCLUDE_PATH_WORK)
+
+  if (NOT MPI_INCLUDE_PATH_WORK)
+    # If all else fails, just search for mpi.h in the normal include
+    # paths.
+    find_path(MPI_INCLUDE_PATH mpi.h
+  HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
+  PATH_SUFFIXES include
+    )
+    set(MPI_INCLUDE_PATH_WORK ${MPI_INCLUDE_PATH})
+  endif (NOT MPI_INCLUDE_PATH_WORK)
+
+  # Extract linker paths from the link command line
+  string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}")
+  set(MPI_LINK_PATH)
+  foreach(LPATH ${MPI_ALL_LINK_PATHS})
+    string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH})
+    string(REGEX REPLACE "//" "/" LPATH ${LPATH})
+    list(APPEND MPI_LINK_PATH ${LPATH})
+  endforeach(LPATH)
+
+  if (NOT MPI_LINK_PATH)
+    if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
+      # The compile command line didn't have any linking paths on it,
+      # but we may have -showme:libdirs. Use it.
+      exec_program(${MPI_COMPILER}
+        ARGS -showme:libdirs
+        OUTPUT_VARIABLE MPI_LINK_PATH
+        RETURN_VALUE MPI_COMPILER_RETURN)
+      separate_arguments(MPI_LINK_PATH)
+    endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
+  endif (NOT MPI_LINK_PATH)
+
+  # Extract linker flags from the link command line
+  string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}")
+  set(MPI_LINK_FLAGS_WORK)
+  foreach(FLAG ${MPI_ALL_LINK_FLAGS})
+    if (MPI_LINK_FLAGS_WORK)
+      set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}")
+    else(MPI_LINK_FLAGS_WORK)
+      set(MPI_LINK_FLAGS_WORK ${FLAG})
+    endif(MPI_LINK_FLAGS_WORK)
+  endforeach(FLAG)
+  if ( MPI_LINK_FLAGS_WORK )
+    string ( REGEX REPLACE "^ " "" MPI_LINK_FLAGS_WORK ${MPI_LINK_FLAGS_WORK} )
+  endif ()
+
+  # Extract the set of libraries to link against from the link command
+  # line
+  string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}")
+
+  # Determine full path names for all of the libraries that one needs
+  # to link against in an MPI program
+  set(MPI_LIBRARIES)
+  foreach(LIB ${MPI_LIBNAMES})
+    string(REGEX REPLACE "^ ?-l" "" LIB ${LIB})
+    set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE)
+    find_library(MPI_LIB ${LIB} HINTS ${MPI_LINK_PATH})
+    if (MPI_LIB)
+      list(APPEND MPI_LIBRARIES ${MPI_LIB})
+    elseif (NOT MPI_FIND_QUIETLY)
+      message(WARNING "Unable to find MPI library ${LIB}")
+    endif ()
+  endforeach(LIB)
+  set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI detection" FORCE)
+
+  # Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and
+  # MPI_EXTRA_LIBRARY.
+  list(LENGTH MPI_LIBRARIES MPI_NUMLIBS)
+  list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED)
+  if (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
+    list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK)
+    set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE)
+  else (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
+    set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE)
+  endif (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
+  if (MPI_NUMLIBS GREATER 1)
+    set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES})
+    list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0)
+    set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE)
+  else (MPI_NUMLIBS GREATER 1)
+    set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE)
+  endif (MPI_NUMLIBS GREATER 1)
+
+  # Set up all of the appropriate cache entries
+  set(MPI_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI compilation flags" FORCE)
+  set(MPI_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI include path" FORCE)
+  set(MPI_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI linking flags" FORCE)
+else (MPI_COMPILE_CMDLINE)
+# No MPI compiler to interogate so attempt to find everything with find functions.
+  find_path(MPI_INCLUDE_PATH mpi.h
+    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
+    PATH_SUFFIXES include Inc
+    )
+
+  # Decide between 32-bit and 64-bit libraries for Microsoft's MPI
+  if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8)
+    set(MS_MPI_ARCH_DIR amd64)
+  else()
+    set(MS_MPI_ARCH_DIR i386)
+  endif()
+
+  find_library(MPI_LIBRARY
+    NAMES mpi mpich msmpi
+    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
+    PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR}
+    )
+
+  find_library(MPI_EXTRA_LIBRARY
+    NAMES mpi++
+    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
+    PATH_SUFFIXES lib
+    DOC "Extra MPI libraries to link against.")
+
+  set(MPI_COMPILE_FLAGS "" CACHE STRING "MPI compilation flags")
+  set(MPI_LINK_FLAGS "" CACHE STRING "MPI linking flags")
+endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+
+# Set up extra variables to conform to
+if (MPI_EXTRA_LIBRARY)
+  set(MPI_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY})
+else (MPI_EXTRA_LIBRARY)
+  set(MPI_LIBRARIES ${MPI_LIBRARY})
+endif (MPI_EXTRA_LIBRARY)
+
+if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+  set(MPI_FOUND TRUE)
+else (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+  set(MPI_FOUND FALSE)
+endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+
+#include("${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake")
+# handle the QUIETLY and REQUIRED arguments
+#find_package_handle_standard_args(MPI DEFAULT_MSG MPI_LIBRARY MPI_INCLUDE_PATH)
+
+mark_as_advanced(MPI_INCLUDE_PATH MPI_COMPILE_FLAGS MPI_LINK_FLAGS MPI_LIBRARY
+  MPI_EXTRA_LIBRARY)
+
+# unset to cleanup namespace
+unset(_MPI_PACKAGE_DIR)
+unset(_MPI_PREFIX_PATH)
+unset(_MPI_BASE_DIR)
@@ -0,0 +1,119 @@
+# - Run Doxygen
+#
+# Adds a doxygen target that runs doxygen to generate the html
+# and optionally the LaTeX API documentation.
+# The doxygen target is added to the doc target as a dependency.
+# i.e.: the API documentation is built with:
+#  make doc
+#
+# USAGE: GLOBAL INSTALL
+#
+# Install it with:
+#  cmake ./ && sudo make install
+# Add the following to the CMakeLists.txt of your project:
+#  include(UseDoxygen OPTIONAL)
+# Optionally copy Doxyfile.in in the directory of CMakeLists.txt and edit it.
+#
+# USAGE: INCLUDE IN PROJECT
+#
+#  set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR})
+#  include(UseDoxygen)
+# Add the Doxyfile.in and UseDoxygen.cmake files to the projects source directory.
+#
+#
+# Variables you may define are:
+#  DOXYFILE_SOURCE_DIR - Path where the Doxygen input files are.
+#  	Defaults to the current source and binary directory.
+#  DOXYFILE_OUTPUT_DIR - Path where the Doxygen output is stored. Defaults to "doc".
+#
+#  DOXYFILE_LATEX - Set to "NO" if you do not want the LaTeX documentation
+#  	to be built.
+#  DOXYFILE_LATEX_DIR - Directory relative to DOXYFILE_OUTPUT_DIR where
+#  	the Doxygen LaTeX output is stored. Defaults to "latex".
+#
+#  DOXYFILE_HTML_DIR - Directory relative to DOXYFILE_OUTPUT_DIR where
+#  	the Doxygen html output is stored. Defaults to "html".
+#
+
+#
+#  Copyright (c) 2009, 2010 Tobias Rautenkranz <tobias@rautenkranz.ch>
+#
+#  Redistribution and use is allowed according to the terms of the New
+#  BSD license.
+#  For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+#
+
+macro(usedoxygen_set_default name value)
+	if(NOT DEFINED "${name}")
+		set("${name}" "${value}")
+	endif()
+endmacro()
+
+find_package(Doxygen)
+
+if(DOXYGEN_FOUND)
+	find_file(DOXYFILE_IN "Doxyfile.in"
+			PATHS "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_ROOT}/Modules/"
+			NO_DEFAULT_PATH)
+	set(DOXYFILE "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile")
+	include(FindPackageHandleStandardArgs)
+	find_package_handle_standard_args(DOXYFILE_IN DEFAULT_MSG "DOXYFILE_IN")
+endif()
+
+if(DOXYGEN_FOUND AND DOXYFILE_IN_FOUND)
+	usedoxygen_set_default(DOXYFILE_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/doc")
+	usedoxygen_set_default(DOXYFILE_HTML_DIR "html")
+	usedoxygen_set_default(DOXYFILE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}\"
+		\"${CMAKE_CURRENT_BINARY_DIR}")
+
+	set_property(DIRECTORY APPEND PROPERTY
+		ADDITIONAL_MAKE_CLEAN_FILES
+		"${DOXYFILE_OUTPUT_DIR}/${DOXYFILE_HTML_DIR}")
+
+	add_custom_target(doxygen
+		COMMAND ${DOXYGEN_EXECUTABLE}
+			${DOXYFILE} 
+		COMMENT "Writing documentation to ${DOXYFILE_OUTPUT_DIR}..."
+		WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
+	## LaTeX
+	set(DOXYFILE_PDFLATEX "NO")
+	set(DOXYFILE_DOT "NO")
+
+	find_package(LATEX)
+	find_program(MAKE_PROGRAM make)
+	if(LATEX_COMPILER AND MAKEINDEX_COMPILER AND MAKE_PROGRAM AND
+			(NOT DEFINED DOXYFILE_LATEX OR DOXYFILE_LATEX STREQUAL "YES"))
+		set(DOXYFILE_LATEX "YES")
+		usedoxygen_set_default(DOXYFILE_LATEX_DIR "latex")
+
+		set_property(DIRECTORY APPEND PROPERTY
+				ADDITIONAL_MAKE_CLEAN_FILES
+				"${DOXYFILE_OUTPUT_DIR}/${DOXYFILE_LATEX_DIR}")
+
+		if(PDFLATEX_COMPILER)
+			set(DOXYFILE_PDFLATEX "YES")
+		endif()
+		if(DOXYGEN_DOT_EXECUTABLE)
+			set(DOXYFILE_DOT "YES")
+		endif()
+
+		add_custom_command(TARGET doxygen
+			POST_BUILD
+			COMMAND ${MAKE_PROGRAM}
+			COMMENT	"Running LaTeX for Doxygen documentation in ${DOXYFILE_OUTPUT_DIR}/${DOXYFILE_LATEX_DIR}..."
+			WORKING_DIRECTORY "${DOXYFILE_OUTPUT_DIR}/${DOXYFILE_LATEX_DIR}")
+	else()
+		set(DOXYGEN_LATEX "NO")
+	endif()
+
+
+	configure_file(${DOXYFILE_IN} Doxyfile IMMEDIATE @ONLY)
+
+	get_target_property(DOC_TARGET doc TYPE)
+	if(NOT DOC_TARGET)
+		add_custom_target(doc)
+	endif()
+
+	add_dependencies(doc doxygen)
+endif()
@@ -0,0 +1,134 @@
+MACRO ( CONFIGURE_LINE_COVERAGE )
+    SET ( COVERAGE_LIBS )
+    IF ( ENABLE_GCOV )
+        ADD_DEFINITIONS ( -fprofile-arcs -ftest-coverage )
+        SET ( COVERAGE_LIBS -lgcov -fprofile-arcs )
+    ENDIF ()
+ENDMACRO ()
+
+
+# Macro to find and configure the MPI libraries
+MACRO ( CONFIGURE_MPI )
+    # Determine if we want to use MPI
+    CHECK_ENABLE_FLAG(USE_MPI 1 )
+    IF ( USE_MPI )
+        # Check if we specified the MPI directory
+        IF ( MPI_DIRECTORY )
+            # Check the provided MPI directory for include files and the mpi executable
+            VERIFY_PATH ( ${MPI_DIRECTORY} )
+            SET ( MPI_INCLUDE_PATH ${MPI_DIRECTORY}/include )
+            VERIFY_PATH ( ${MPI_INCLUDE_PATH} )
+            IF ( NOT EXISTS ${MPI_INCLUDE_PATH}/mpi.h )
+                MESSAGE ( FATAL_ERROR "mpi.h not found in ${MPI_INCLUDE_PATH}/include" )
+            ENDIF ()
+            INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} )
+            SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} )
+            IF ( MPIEXEC ) 
+                # User specified the MPI command directly, use as is
+            ELSEIF ( MPIEXEC_CMD )
+                # User specified the name of the MPI executable
+                SET ( MPIEXEC ${MPI_DIRECTORY}/bin/${MPIEXEC_CMD} )
+                IF ( NOT EXISTS ${MPIEXEC} )
+                    MESSAGE ( FATAL_ERROR "${MPIEXEC_CMD} not found in ${MPI_DIRECTORY}/bin" )
+                ENDIF ()
+            ELSE ()
+                # Search for the MPI executable in the current directory
+                FIND_PROGRAM ( MPIEXEC  NAMES mpiexec mpirun lamexec  PATHS ${MPI_DIRECTORY}/bin  NO_DEFAULT_PATH )
+                IF ( NOT MPIEXEC )
+                    MESSAGE ( FATAL_ERROR "Could not locate mpi executable" )
+                ENDIF()
+            ENDIF ()
+            # Set MPI flags
+            IF ( NOT MPIEXEC_NUMPROC_FLAG )
+                SET( MPIEXEC_NUMPROC_FLAG "-np" )
+            ENDIF()
+        ELSEIF ( MPI_COMPILER )
+            # The mpi compiler should take care of everything
+        ELSE()
+            # Perform the default search for MPI
+            INCLUDE ( FindMPI )
+            IF ( NOT MPI_FOUND )
+                MESSAGE ( FATAL_ERROR "Did not find MPI" )
+            ENDIF ()
+            INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} )
+            SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} )
+        ENDIF()
+        # Check if we need to use MPI for serial tests
+        CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 )
+        # Set the definitions
+        ADD_DEFINITIONS ( "-D USE_MPI" )  
+        MESSAGE ( "Using MPI" )
+        MESSAGE ( "  MPIEXEC = ${MPIEXEC}" )
+        MESSAGE ( "  MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" )
+        MESSAGE ( "  MPI_INCLUDE = ${MPI_INCLUDE}" )
+        MESSAGE ( "  MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
+        MESSAGE ( "  MPI_LIBRARIES = ${MPI_LIBRARIES}" )
+    ELSE()
+        SET( USE_MPI_FOR_SERIAL_TESTS 0 )
+        SET( MPIEXEC "" )
+        SET( MPIEXEC_NUMPROC_FLAG "" )
+        SET( MPI_INCLUDE "" )
+        SET( MPI_LINK_FLAGS "" )
+        SET( MPI_LIBRARIES "" )
+        MESSAGE ( "Not using MPI, all parallel tests will be disabled" )
+    ENDIF()
+ENDMACRO ()
+
+
+# Macro to configure system-specific libraries and flags
+MACRO ( CONFIGURE_SYSTEM )
+    # First check/set the compile mode
+    IF( NOT CMAKE_BUILD_TYPE )
+        MESSAGE(FATAL_ERROR "CMAKE_BUILD_TYPE is not set")
+    ENDIF()
+    # Remove extra library links
+    # Get the compiler
+    SET_COMPILER ()
+    # Add the static flag if necessary
+    CHECK_ENABLE_FLAG( USE_EXT_STATIC 0 )
+    IF ( USE_EXT_STATIC )
+        SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "-static")    # Add static flag
+        SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "-static")  # Add static flag
+    ENDIF()
+    # Add system dependent flags
+    IF ( USING_MICROSOFT )
+        # Windows specific system libraries
+        SET( SYSTEM_PATHS "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib/x64" 
+                          "C:/Program Files (x86)/Microsoft Visual Studio 8/VC/PlatformSDK/Lib/AMD64" )
+        FIND_LIBRARY ( PSAPI_LIB    NAMES Psapi    PATHS ${SYSTEM_PATHS}  NO_DEFAULT_PATH )
+        FIND_LIBRARY ( DBGHELP_LIB  NAMES DbgHelp  PATHS ${SYSTEM_PATHS}  NO_DEFAULT_PATH )
+        SET( SYSTEM_LIBS ${PSAPI_LIB} ${DBGHELP_LIB} )
+        MESSAGE("System libs: ${SYSTEM_LIBS}")
+    ELSEIF( ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" )
+        # Linux specific system libraries
+        CHECK_C_COMPILER_FLAG("-rdynamic" RESULT)
+        IF(RESULT)
+            SET( SYSTEM_LIBS "-lpthread -lz -ldl -rdynamic" )
+        ELSE()
+            SET( SYSTEM_LIBS "-lpthread -lz -ldl" )
+        ENDIF()
+        IF ( USING_GCC )
+            SET( SYSTEM_LIBS ${SYSTEM_LIBS} "-lgfortran" )
+            SET(CMAKE_C_FLAGS   " ${CMAKE_C_FLAGS} -fPIC" )
+            SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -fPIC" )
+        ENDIF()
+    ELSEIF( ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" )
+        # Max specific system libraries
+        SET( SYSTEM_LIBS "-lz -ldl" )
+    ELSEIF( ${CMAKE_SYSTEM_NAME} STREQUAL "Generic" )
+        # Generic system libraries
+    ELSE()
+        MESSAGE( FATAL_ERROR "OS not detected" )
+    ENDIF()
+    # Set the compile flags based on the build
+    SET_COMPILE_FLAGS()
+ENDMACRO ()
+
+
+# Macro to configure AtomicModel-specific options
+MACRO ( CONFIGURE_LBPM )
+    # Set the maximum number of processors for the tests
+    IF ( NOT TEST_MAX_PROCS )
+        SET( TEST_MAX_PROCS 32 )
+    ENDIF()
+ENDMACRO ()
@@ -0,0 +1,490 @@
+
+
+
+INCLUDE(CheckCSourceCompiles)
+
+
+# Add a package to the LBPM library
+MACRO( ADD_LBPM_LIBRARY PACKAGE )
+    #INCLUDE_DIRECTORIES ( ${LBPM_INSTALL_DIR}/include/${PACKAGE} )
+    ADD_SUBDIRECTORY( ${PACKAGE} )
+ENDMACRO()
+
+
+# Add an LBPM executable
+MACRO( ADD_LBPM_EXECUTABLE PACKAGE )
+    ADD_SUBDIRECTORY( ${PACKAGE} )
+ENDMACRO()
+
+
+# Initialize a package
+MACRO (BEGIN_PACKAGE_CONFIG PACKAGE)
+    SET( HEADERS "" )
+    SET( CXXSOURCES "" )
+    SET( CSOURCES "" )
+    SET( FSOURCES "" )
+    SET( M4FSOURCES "" )
+    SET( SOURCES "" )
+    SET( CURPACKAGE ${PACKAGE} )
+ENDMACRO ()
+
+
+# Find the source files
+MACRO (FIND_FILES)
+    # Find the C/C++ headers
+    SET( T_HEADERS "" )
+    FILE( GLOB T_HEADERS "*.h" "*.hh" "*.hpp" "*.I" )
+    # Find the C sources
+    SET( T_CSOURCES "" )
+    FILE( GLOB T_CSOURCES "*.c" )
+    # Find the C++ sources
+    SET( T_CXXSOURCES "" )
+    FILE( GLOB T_CXXSOURCES "*.cc" "*.cpp" "*.cxx" "*.C" )
+    # Add all found files to the current lists
+    SET( HEADERS ${HEADERS} ${T_HEADERS} )
+    SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} )
+    SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
+    SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} )
+ENDMACRO()
+
+
+# Find the source files
+MACRO (FIND_FILES_PATH IN_PATH)
+    # Find the C/C++ headers
+    SET( T_HEADERS "" )
+    FILE( GLOB T_HEADERS "${IN_PATH}/*.h" "${IN_PATH}/*.hh" "${IN_PATH}/*.hpp" "${IN_PATH}/*.I" )
+    # Find the C sources
+    SET( T_CSOURCES "" )
+    FILE( GLOB T_CSOURCES "${IN_PATH}/*.c" )
+    # Find the C++ sources
+    SET( T_CXXSOURCES "" )
+    FILE( GLOB T_CXXSOURCES "${IN_PATH}/*.cc" "${IN_PATH}/*.cpp" "${IN_PATH}/*.cxx" "${IN_PATH}/*.C" )
+    # Add all found files to the current lists
+    SET( HEADERS ${HEADERS} ${T_HEADERS} )
+    SET( CXXSOURCES ${CXXSOURCES} ${T_CXXSOURCES} )
+    SET( CSOURCES ${CSOURCES} ${T_CSOURCES} )
+    SET( SOURCES ${SOURCES} ${T_CXXSOURCES} ${T_CSOURCES} )
+ENDMACRO()
+
+
+# Add a subdirectory
+MACRO( ADD_PACKAGE_SUBDIRECTORY SUBDIR )
+    SET( FULLSUBDIR ${CMAKE_CURRENT_SOURCE_DIR}/${SUBDIR} )
+    FIND_FILES_PATH( ${SUBDIR} )
+    FILE( GLOB HFILES RELATIVE ${FULLSUBDIR} ${SUBDIR}/*.h ${SUBDIR}/*.hh ${SUBDIR}/*.hpp  ${SUBDIR}/*.I )
+    FOREACH( HFILE ${HFILES} )
+        CONFIGURE_FILE( ${FULLSUBDIR}/${HFILE} ${LBPM_INSTALL_DIR}/include/${SUBDIR}/${HFILE} COPYONLY )
+        INCLUDE_DIRECTORIES( ${FULLSUBDIR} )
+    ENDFOREACH()
+    ADD_SUBDIRECTORY( ${SUBDIR} )
+ENDMACRO()
+
+
+# Install a package
+MACRO( INSTALL_LBPM_TARGET PACKAGE )
+    # Find all files in the current directory
+    FIND_FILES()
+    # Copy the header files to the include path
+    FILE( GLOB HFILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.hh ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp ${CMAKE_CURRENT_SOURCE_DIR}/*.I )
+    FOREACH( HFILE ${HFILES} )
+        #CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/${HFILE} ${LBPM_INSTALL_DIR}/include/${CURPACKAGE}/${HFILE} COPYONLY )
+        CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/${HFILE} ${LBPM_INSTALL_DIR}/include/${HFILE} COPYONLY )
+    ENDFOREACH()
+    # Add the library
+    ADD_LIBRARY( ${PACKAGE} ${LIB_TYPE} ${SOURCES} )
+    SET( TEST_DEP_LIST ${PACKAGE} ${TEST_DEP_LIST} )
+    TARGET_LINK_LIBRARIES( ${PACKAGE} ${COVERAGE_LIBS} ${SYSTEM_LIBS} ${LDLIBS} )
+    TARGET_LINK_LIBRARIES( ${PACKAGE} ${LAPACK_LIBS} ${BLAS_LIBS} )
+    IF ( USE_MPI )
+        TARGET_LINK_LIBRARIES( ${PACKAGE} ${MPI_LIBRARIES} )
+    ENDIF()
+    TARGET_LINK_LIBRARIES( ${PACKAGE} ${COVERAGE_LIBS} ${SYSTEM_LIBS} ${LDLIBS} )
+    # Install the package
+    INSTALL( TARGETS ${PACKAGE} DESTINATION ${LBPM_INSTALL_DIR}/lib )
+    INSTALL( FILES ${HFILES} DESTINATION ${LBPM_INSTALL_DIR}/include )
+    # Clear the sources
+    SET( HEADERS "" )
+    SET( CSOURCES "" )
+    SET( CXXSOURCES "" )
+ENDMACRO()
+
+
+# Macro to verify that a variable has been set
+MACRO( VERIFY_VARIABLE VARIABLE_NAME )
+    IF( NOT ${VARIABLE_NAME} )
+        MESSAGE( FATAL_ERROR "PLease set: " ${VARIABLE_NAME} )
+    ENDIF()
+ENDMACRO()
+
+
+# Macro to verify that a path has been set
+MACRO( VERIFY_PATH PATH_NAME )
+    IF( NOT EXISTS "${PATH_NAME}" )
+        MESSAGE( FATAL_ERROR "Path does not exist: " ${PATH_NAME} )
+    ENDIF()
+ENDMACRO()
+
+
+# Macro to identify the compiler
+MACRO( SET_COMPILER )
+    # SET the C/C++ compiler
+    IF( CMAKE_COMPILE_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX )
+        SET( USING_GCC TRUE )
+        MESSAGE("Using gcc")
+    ELSEIF( MSVC OR MSVC_IDE OR MSVC60 OR MSVC70 OR MSVC71 OR MSVC80 OR CMAKE_COMPILER_2005 OR MSVC90 OR MSVC10 )
+        IF( NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
+            MESSAGE( FATAL_ERROR "Using microsoft compilers on non-windows system?" )
+        ENDIF()
+        SET( USING_MICROSOFT TRUE )
+        MESSAGE("Using Microsoft")
+    ELSEIF( (${CMAKE_C_COMPILER_ID} MATCHES "Intel") OR (${CMAKE_CXX_COMPILER_ID} MATCHES "Intel") ) 
+        SET(USING_ICC TRUE)
+        MESSAGE("Using icc")
+    ELSEIF( ${CMAKE_C_COMPILER_ID} MATCHES "PGI")
+        SET(USING_PGCC TRUE)
+        MESSAGE("Using pgCC")
+    ELSE()
+        SET(USING_DEFAULT TRUE)
+        MESSAGE("${CMAKE_C_COMPILER_ID}")
+        MESSAGE("Unknown C/C++ compiler, default flags will be used")
+    ENDIF()
+ENDMACRO()
+
+
+# Macro to set the proper warnings
+MACRO ( SET_WARNINGS )
+  IF ( USING_GCC )
+    ## Add gcc specific compiler options
+    ##    -Wno-reorder:  warning: "" will be initialized after "" when initialized here
+    #SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -Wall ") 
+    #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall ")
+  ELSEIF ( USING_MICROSOFT )
+    # Add Microsoft specifc compiler options
+    SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} /D _SCL_SECURE_NO_WARNINGS /D _CRT_SECURE_NO_WARNINGS /D _ITERATOR_DEBUG_LEVEL=0" )
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D _SCL_SECURE_NO_WARNINGS /D _CRT_SECURE_NO_WARNINGS /D _ITERATOR_DEBUG_LEVEL=0" )
+  ELSEIF ( USING_ICC )
+    ## Add Intel specifc compiler options
+    #SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -Wall" )
+    #SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall" )
+    ## Disable warnings that I think are irrelavent (may need to be revisited)
+    #SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -wd383 -wd981" )
+    #SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -wd383 -wd981" )
+  ELSEIF ( USING_DEFAULT )
+    ## Add default compiler options
+    #SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -Wall")
+    #SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -Wall")
+  ENDIF ()
+ENDMACRO ()
+
+
+# Macro to add user compile flags
+MACRO( ADD_USER_FLAGS )
+    SET(CMAKE_C_FLAGS   " ${CMAKE_C_FLAGS} ${CFLAGS} ${LDFLAGS}" )
+    SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} ${CXXFLAGS} ${LDFLAGS}" )
+ENDMACRO()
+
+
+# Macro to set the flags for debug mode
+MACRO( SET_COMPILE_FLAGS )
+    ADD_USER_FLAGS()
+    IF ( ${CMAKE_BUILD_TYPE} STREQUAL "Debug" )
+        IF ( NOT DISABLE_GXX_DEBUG )
+            SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -D_GLIBCXX_DEBUG" )
+            SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -D_GLIBCXX_DEBUG" )
+            SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -D_GLIBCXX_DEBUG_PEDANTIC" )
+            SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -D_GLIBCXX_DEBUG_PEDANTIC" )
+        ENDIF ()
+        IF ( USING_MICROSOFT )
+            SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -DDEBUG /DEBUG /Od" )
+            SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -DDEBUG /DEBUG /Od" )
+            SET(CONFIGURATION Debug )
+        ELSE()
+            SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -DDEBUG -g -O0" )
+            SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -DDEBUG -g -O0" )
+        ENDIF()
+    ELSEIF ( ${CMAKE_BUILD_TYPE} STREQUAL "Release" )
+        IF ( USING_MICROSOFT )
+            SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} /O2" )
+            SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} /O2" )
+            SET(CONFIGURATION Release )
+        ELSE()
+            SET(CMAKE_C_FLAGS     " ${CMAKE_C_FLAGS} -O2" )
+            SET(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -O2" )
+        ENDIF()
+    ELSE()
+        MESSAGE(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
+    ENDIF()
+    SET_WARNINGS()
+ENDMACRO()
+
+
+
+
+# Macro to add the dependencies and libraries to an executable
+MACRO( ADD_LBPM_EXE_DEP EXE )
+    # Add the package dependencies
+    IF( LBPM_TEST_LIB_EXISTS )
+        ADD_DEPENDENCIES ( ${EXE} ${PACKAGE_TEST_LIB} )
+        TARGET_LINK_LIBRARIES ( ${EXE} ${PACKAGE_TEST_LIB} )
+    ENDIF()
+    # Add the executable to the dependencies of check and build-test
+    ADD_DEPENDENCIES( check ${EXE} )
+    ADD_DEPENDENCIES( build-test ${EXE} )
+    # Add the libraries
+    TARGET_LINK_LIBRARIES( ${EXE} ${LBPM_LIBS} )
+    # Add external libraries
+    IF ( USE_MPI )
+        TARGET_LINK_LIBRARIES( ${EXE} ${MPI_LINK_FLAGS} ${MPI_LIBRARIES} )
+    ENDIF()
+    TARGET_LINK_LIBRARIES( ${EXE} ${LAPACK_LIBS} ${BLAS_LIBS} )
+    TARGET_LINK_LIBRARIES( ${EXE} ${COVERAGE_LIBS} ${SYSTEM_LIBS} ${LDLIBS} )
+ENDMACRO()
+
+
+# Add a executable
+MACRO( INSTALL_LBPM_EXE EXE )
+    SET( SOURCES ${EXE}.cpp )
+    ADD_EXECUTABLE( ${EXE} ${SOURCES} )
+    ADD_LBPM_EXE_DEP( ${EXE} )
+    INSTALL( TARGETS ${EXE} DESTINATION ${LBPM_INSTALL_DIR}/bin )
+ENDMACRO()
+
+
+# Add a provisional test
+FUNCTION( ADD_LBPM_PROVISIONAL_TEST EXEFILE )
+    # Check if we actually want to add the test
+    SET( EXCLUDE_TESTS_FROM_ALL 0 )
+    # Check if test has already been added
+    GET_TARGET_PROPERTY(tmp ${EXEFILE} LOCATION)
+    IF ( NOT tmp )
+        # The target has not been added
+        SET( CXXFILE ${EXEFILE}.cpp )
+        SET( TESTS_SO_FAR ${TESTS_SO_FAR} ${EXEFILE} )
+        IF ( NOT EXCLUDE_TESTS_FROM_ALL )
+            ADD_EXECUTABLE( ${EXEFILE} ${CXXFILE} )
+        ELSE()
+            ADD_EXECUTABLE( ${EXEFILE} EXCLUDE_FROM_ALL ${CXXFILE} )
+        ENDIF()
+        ADD_LBPM_EXE_DEP( ${EXEFILE} )
+    ELSEIF( ${tmp} STREQUAL "${CMAKE_CURRENT_BINARY_DIR}/${EXEFILE}" )
+        # The correct target has already been added
+    ELSEIF( ${tmp} STREQUAL "${CMAKE_CURRENT_BINARY_DIR}/${EXEFILE}.exe" )
+        # The correct target has already been added
+    ELSEIF( ${tmp} STREQUAL "${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/${EXEFILE}.exe" )
+        # The correct target has already been added
+    ELSEIF( ${tmp} STREQUAL "${CMAKE_CURRENT_BINARY_DIR}/$(OutDir)/${EXEFILE}.exe" )
+        # The correct target has already been added
+    ELSE()
+        # We are trying to add 2 different tests with the same name
+        MESSAGE ( "Existing test: ${tmp}" )
+        MESSAGE ( "New test:      ${CMAKE_CURRENT_BINARY_DIR}/${EXEFILE}" )
+        MESSAGE ( FATAL_ERROR "Trying to add 2 different tests with the same name" )
+    ENDIF()
+ENDFUNCTION()
+
+
+# Macro to create the test name
+MACRO( CREATE_TEST_NAME TEST ${ARGN} )
+    SET( TESTNAME "${TEST}" )
+    FOREACH( tmp ${ARGN} )
+        SET( TESTNAME "${TESTNAME}--${tmp}")
+    endforeach()
+    # STRING(REGEX REPLACE "--" "-" TESTNAME ${TESTNAME} )
+ENDMACRO()
+
+
+# Add a executable as a test
+FUNCTION( ADD_LBPM_TEST EXEFILE ${ARGN} )
+    ADD_LBPM_PROVISIONAL_TEST ( ${EXEFILE} )
+    CREATE_TEST_NAME( ${EXEFILE} ${ARGN} )
+    GET_TARGET_PROPERTY(EXE ${EXEFILE} LOCATION)
+    STRING(REGEX REPLACE "\\$\\(Configuration\\)" "${CONFIGURATION}" EXE "${EXE}" )
+    IF ( USE_EXT_MPI_FOR_SERIAL_TESTS )
+        ADD_TEST( ${TESTNAME} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${EXE} ${ARGN} )
+    ELSE()
+        ADD_TEST( ${TESTNAME} ${CMAKE_CURRENT_BINARY_DIR}/${EXEFILE} ${ARGN} )
+    ENDIF()
+    SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION ".*FAILED.*" PROCESSORS 1 )
+ENDFUNCTION()
+
+
+# Add a executable as a weekly test
+FUNCTION( ADD_LBPM_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
+    ADD_LBPM_PROVISIONAL_TEST ( ${EXEFILE} )
+    GET_TARGET_PROPERTY(EXE ${EXEFILE} LOCATION)
+    STRING(REGEX REPLACE "\\$\\(Configuration\\)" "${CONFIGURATION}" EXE "${EXE}" )
+    IF( ${PROCS} STREQUAL "1" )
+        CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
+        IF( USE_EXT_MPI_FOR_SERIAL_TESTS )
+            ADD_TEST( ${TESTNAME} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${EXE} ${ARGN} )
+        ELSE()
+            ADD_TEST( ${TESTNAME} ${CMAKE_CURRENT_BINARY_DIR}/${EXEFILE} ${ARGN} )
+        ENDIF()
+    ELSEIF( USE_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
+        CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} )
+        ADD_TEST( ${TESTNAME} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} ${EXE} ${ARGN} )
+    ENDIF()
+    SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION ".*FAILED.*" PROCESSORS ${PROCS} )
+ENDFUNCTION()
+
+
+# Add a executable as a parallel test
+FUNCTION( ADD_LBPM_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
+    ADD_LBPM_PROVISIONAL_TEST ( ${EXEFILE} )
+    GET_TARGET_PROPERTY(EXE ${EXEFILE} LOCATION)
+    STRING(REGEX REPLACE "\\$\\(Configuration\\)" "${CONFIGURATION}" EXE "${EXE}" )
+    IF ( USE_MPI )
+        CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs" ${ARGN} )
+        ADD_TEST( ${TESTNAME} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} ${EXE} ${ARGN} )
+        SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION ".*FAILED.*" PROCESSORS ${PROCS} )
+    ENDIF()
+ENDFUNCTION()
+
+
+# Add a parallel test on 1, 2, and 4 processors
+MACRO( ADD_LBPM_TEST_1_2_4 EXENAME ${ARGN} )
+    ADD_LBPM_TEST ( ${EXENAME} ${ARGN} )
+    ADD_LBPM_TEST_PARALLEL ( ${EXENAME} 2 ${ARGN} )
+    ADD_LBPM_TEST_PARALLEL ( ${EXENAME} 4 ${ARGN} )
+ENDMACRO()
+
+
+# Add a parallel test that may use both MPI and threads
+# This allows us to correctly compute the number of processors used by the test
+MACRO( ADD_LBPM_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
+    ADD_LBPM_PROVISIONAL_TEST( ${EXEFILE} )
+    GET_TARGET_PROPERTY(EXE ${EXEFILE} LOCATION)
+    STRING(REGEX REPLACE "\\$\\(Configuration\\)" "${CONFIGURATION}" EXE "${EXE}" )
+    CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_${THREADS}threads" ${ARGN} )
+    MATH( EXPR TOT_PROCS "${PROCS} * ${THREADS}" )
+    IF ( ${TOT_PROCS} GREATER ${TEST_MAX_PROCS} )
+        MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS}")
+    ELSEIF ( ( ${PROCS} STREQUAL "1" ) AND NOT USE_EXT_MPI_FOR_SERIAL_TESTS )
+        ADD_TEST ( ${TESTNAME} ${CMAKE_CURRENT_BINARY_DIR}/${EXEFILE} ${ARGN} )
+    ELSEIF ( USE_MPI )
+        ADD_TEST ( ${TESTNAME} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} ${EXE} ${ARGN} )
+    ENDIF()
+    SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION ".*FAILED.*" PROCESSORS ${TOT_PROCS} )
+ENDMACRO()
+
+
+# Macro to check if a flag is enabled
+MACRO ( CHECK_ENABLE_FLAG FLAG DEFAULT )
+    IF( NOT DEFINED ${FLAG} )
+        SET( ${FLAG} ${DEFAULT} )
+    ELSEIF( ${FLAG}  STREQUAL "" )
+        SET( ${FLAG} ${DEFAULT} )
+    ELSEIF( ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
+        SET( ${FLAG} 0 )
+    ELSEIF( ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
+        SET( ${FLAG} 1 )
+    ELSE()
+        MESSAGE( "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
+    ENDIF ()
+ENDMACRO()
+
+
+# Macro to check if a compiler flag is valid
+MACRO (CHECK_C_COMPILER_FLAG _FLAG _RESULT)
+    SET(SAFE_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}")
+    SET(CMAKE_REQUIRED_DEFINITIONS "${_FLAG}")
+    CHECK_C_SOURCE_COMPILES("int main() { return 0;}" ${_RESULT}
+        # Some compilers do not fail with a bad flag
+        FAIL_REGEX "error: bad value (.*) for .* switch"       # GNU
+        FAIL_REGEX "argument unused during compilation"        # clang
+        FAIL_REGEX "is valid for .* but not for C"             # GNU
+        FAIL_REGEX "unrecognized .*option"                     # GNU
+        FAIL_REGEX "ignoring unknown option"                   # MSVC
+        FAIL_REGEX "[Uu]nknown option"                         # HP
+        FAIL_REGEX "[Ww]arning: [Oo]ption"                     # SunPro
+        FAIL_REGEX "command option .* is not recognized"       # XL
+        FAIL_REGEX "WARNING: unknown flag:"                    # Open64
+        FAIL_REGEX " #10159: "                                 # ICC
+    )
+    SET(CMAKE_REQUIRED_DEFINITIONS "${SAFE_CMAKE_REQUIRED_DEFINITIONS}")
+ENDMACRO(CHECK_C_COMPILER_FLAG)
+
+
+# Macro to add a latex file to the build
+MACRO (ADD_LATEX_DOCS FILE)
+    GET_FILENAME_COMPONENT(LATEX_TARGET ${FILE} NAME_WE)
+    ADD_CUSTOM_TARGET( 
+        ${LATEX_TARGET}_pdf 
+        ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/${FILE} ${CMAKE_CURRENT_BINARY_DIR}/.
+        COMMAND pdflatex -interaction=batchmode -draftmode ${FILE}
+        #COMMAND bibtex -terse ${LATEX_TARGET}
+        COMMAND pdflatex -interaction=batchmode ${FILE}
+        SOURCES ${FILE}
+    )
+    ADD_CUSTOM_COMMAND( 
+        TARGET ${LATEX_TARGET}_pdf 
+        POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${LATEX_TARGET}.pdf ${AMR_MHD_INSTALL_DIR}/doc/.
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+    ADD_DEPENDENCIES( latex_docs ${LATEX_TARGET}_pdf )
+ENDMACRO()
+
+
+# add custom target distclean
+# cleans and removes cmake generated files etc.
+MACRO( ADD_DISTCLEAN )
+    SET(DISTCLEANED
+        cmake.depends
+        cmake.check_depends
+        CMakeCache.txt
+        CMakeFiles
+        CMakeTmp
+        cmake.check_cache
+        *.cmake
+        compile.log
+        Doxyfile
+        Makefile
+        core core.*
+        DartConfiguration.tcl
+        install_manifest.txt
+        Testing
+        include
+        doc
+        lib
+        tests
+        bin
+        liblbpm-wia.a
+        liblbpm-wia.so
+        cpu
+        gpu
+    )
+    ADD_CUSTOM_TARGET (distclean @echo cleaning for source distribution)
+    IF (UNIX)
+        ADD_CUSTOM_COMMAND(
+            DEPENDS clean
+            COMMENT "distribution clean"
+            COMMAND rm
+            ARGS    -Rf ${DISTCLEANED}
+            TARGET  distclean
+        )
+    ELSE()
+        SET( DISTCLEANED
+            ${DISTCLEANED}
+            *.vcxproj*
+            ipch
+            x64
+            atomic.lib
+            atomic.pdb
+        )
+        FILE(WRITE  ${CMAKE_CURRENT_BINARY_DIR}/distclean.bat "del /s /q /f " )
+        FOREACH (fileToDelete ${DISTCLEANED})
+            FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/distclean.bat "${fileToDelete} " )
+        ENDFOREACH ()
+        FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/distclean.bat "\n" )
+        FOREACH (fileToDelete ${DISTCLEANED})
+            FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/distclean.bat "for /d %%x in (${fileToDelete}) do rd /s /q \"%%x\"\n" )
+        ENDFOREACH ()
+        ADD_CUSTOM_COMMAND(
+            DEPENDS clean
+            COMMENT "distribution clean"
+            COMMAND distclean.bat & del /s/q/f distclean.bat
+            TARGET  distclean
+        )
+    ENDIF()
+ENDMACRO()
@@ -1,756 +0,0 @@
-#include <math.h>
-
-inline void InitDenColor(char *ID, double *Den, double *Phi, double das, double dbs, int N)
-{
-	int n;
-	for (n=0; n<N; n++){
-		
-		if ( ID[n] == 1){
-			Den[2*n] = 1.0;
-			Den[2*n+1] = 0.0;
-			Phi[n] = 1.0;
-		}
-		else if ( ID[n] == 2){
-			Den[2*n] = 0.0;
-			Den[2*n+1] = 1.0;
-			Phi[n] = -1.0;
-		}
-		else{
-			Den[2*n] = das;
-			Den[2*n+1] = dbs;
-			Phi[n] = (das-dbs)/(das+dbs);
-		}
-	}
-}
-inline void InitD3Q19(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz)
-{
-	int n,N;
-	N = Nx*Ny*Nz;
-	
-	for (n=0; n<N; n++){
-		if (ID[n] > 0){
-			f_even[n] = 0.3333333333333333;
-			f_odd[n] = 0.055555555555555555;		//double(100*n)+1.f;
-			f_even[N+n] = 0.055555555555555555;	//double(100*n)+2.f;
-			f_odd[N+n] = 0.055555555555555555;	//double(100*n)+3.f;
-			f_even[2*N+n] = 0.055555555555555555;	//double(100*n)+4.f;
-			f_odd[2*N+n] = 0.055555555555555555;	//double(100*n)+5.f;
-			f_even[3*N+n] = 0.055555555555555555;	//double(100*n)+6.f;
-			f_odd[3*N+n] = 0.0277777777777778;   //double(100*n)+7.f;
-			f_even[4*N+n] = 0.0277777777777778;   //double(100*n)+8.f;
-			f_odd[4*N+n] = 0.0277777777777778;   //double(100*n)+9.f;
-			f_even[5*N+n] = 0.0277777777777778;  //double(100*n)+10.f;
-			f_odd[5*N+n] = 0.0277777777777778;  //double(100*n)+11.f;
-			f_even[6*N+n] = 0.0277777777777778;  //double(100*n)+12.f;
-			f_odd[6*N+n] = 0.0277777777777778;  //double(100*n)+13.f;
-			f_even[7*N+n] = 0.0277777777777778;  //double(100*n)+14.f;
-			f_odd[7*N+n] = 0.0277777777777778;  //double(100*n)+15.f;
-			f_even[8*N+n] = 0.0277777777777778;  //double(100*n)+16.f;
-			f_odd[8*N+n] = 0.0277777777777778;  //double(100*n)+17.f;
-			f_even[9*N+n] = 0.0277777777777778;  //double(100*n)+18.f;
-		}
-		else{
-			for(int q=0; q<9; q++){
-				f_even[q*N+n] = -1.0;
-				f_odd[q*N+n] = -1.0;
-			}
-			f_even[9*N+n] = -1.0;
-		}
-	}
-}
-
-inline void Compute_VELOCITY(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz)
-{
-	int n,N;
-	// distributions
-	double f1,f2,f3,f4,f5,f6,f7,f8,f9;
-	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
-	double vx,vy,vz;
-	
-	N = Nx*Ny*Nz;
-	
-	for (n=0; n<N; n++){
-		
-		if (ID[n] > 0){
-			//........................................................................
-			// Registers to store the distributions
-			//........................................................................
-			f2 = disteven[N+n];
-			f4 = disteven[2*N+n];
-			f6 = disteven[3*N+n];
-			f8 = disteven[4*N+n];
-			f10 = disteven[5*N+n];
-			f12 = disteven[6*N+n];
-			f14 = disteven[7*N+n];
-			f16 = disteven[8*N+n];
-			f18 = disteven[9*N+n];
-			//........................................................................
-			f1 = distodd[n];
-			f3 = distodd[1*N+n];
-			f5 = distodd[2*N+n];
-			f7 = distodd[3*N+n];
-			f9 = distodd[4*N+n];
-			f11 = distodd[5*N+n];
-			f13 = distodd[6*N+n];
-			f15 = distodd[7*N+n];
-			f17 = distodd[8*N+n];
-			//.................Compute the velocity...................................
-			vx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
-			vy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
-			vz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
-			//..................Write the velocity.....................................
-			vel[n] = vx;
-			vel[N+n] = vy;
-			vel[2*N+n] = vz;
-			//........................................................................
-			
-		}
-	}
-}
-
-//*************************************************************************
-//*************************************************************************
-inline  void PressureBC_inlet(double *disteven, double *distodd, double din,
-							  int Nx, int Ny, int Nz)
-{
-	int n,N;
-	// distributions
-	double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
-	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
-	double uz;
-	
-	N = Nx*Ny*Nz;
-	
-	for (n=0; n<Nx*Ny; n++){
-		
-		//........................................................................
-		// Read distributions from "opposite" memory convention
-		//........................................................................
-		//........................................................................
-		f1 = distodd[n];
-		f3 = distodd[N+n];
-		f5 = distodd[2*N+n];
-		f7 = distodd[3*N+n];
-		f9 = distodd[4*N+n];
-		f11 = distodd[5*N+n];
-		f13 = distodd[6*N+n];
-		f15 = distodd[7*N+n];
-		f17 = distodd[8*N+n];
-		//........................................................................
-		f0 = disteven[n];
-		f2 = disteven[N+n];
-		f4 = disteven[2*N+n];
-		f6 = disteven[3*N+n];
-		f8 = disteven[4*N+n];
-		f10 = disteven[5*N+n];
-		f12 = disteven[6*N+n];
-		f14 = disteven[7*N+n];
-		f16 = disteven[8*N+n];
-		f18 = disteven[9*N+n];
-		//...................................................
-		//........Determine the intlet flow velocity.........
-		//			uz = -1 + (f0+f3+f4+f1+f2+f7+f8+f10+f9
-		//					   + 2*(f5+f15+f18+f11+f14))/din;
-		//........Set the unknown distributions..............
-		//			f6 = f5 - 0.3333333333333333*din*uz;
-		//			f16 = f15 - 0.1666666666666667*din*uz;
-		//			f17 = f16 - f3 + f4-f15+f18-f7+f8-f10+f9;
-		//			f12= 0.5*(-din*uz+f5+f15+f18+f11+f14-f6-f16-
-		//					  f17+f1-f2-f14+f11+f7-f8-f10+f9);
-		//			f13= -din*uz+f5+f15+f18+f11+f14-f6-f16-f17-f12;
-		
-		// Determine the outlet flow velocity
-		uz = 1.0 - (f0+f4+f3+f2+f1+f8+f7+f9+ f10 +
-					2*(f5+ f15+f18+f11+f14))/din;
-		// Set the unknown distributions:
-        f6 = f5 + 0.3333333333333333*din*uz;
-        f16 = f15 + 0.1666666666666667*din*uz;
-        f17 = f16 + f4 - f3-f15+f18+f8-f7	+f9-f10;
-        f12= (din*uz+f5+ f15+f18+f11+f14-f6-f16-f17-f2+f1-f14+f11-f8+f7+f9-f10)*0.5;
-        f13= din*uz+f5+ f15+f18+f11+f14-f6-f16-f17-f12;
-		
-		//........Store in "opposite" memory location..........
-		disteven[3*N+n] = f6;
-		disteven[6*N+n] = f12;
-		distodd[6*N+n] = f13;
-		disteven[8*N+n] = f16;
-		distodd[8*N+n] = f17;
-		//...................................................
-	}
-}
-
-inline  void PressureBC_outlet(double *disteven, double *distodd, double dout,
-							   int Nx, int Ny, int Nz, int S, int outlet)
-{
-	int n,N;
-	// distributions
-	double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
-	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
-	double uz;
-	
-	N = Nx*Ny*Nz;
-	
-	// Loop over the boundary - threadblocks delineated by start...finish
-	for ( n=0; n<N; n++){
-		
-		//........................................................................
-		// Read distributions from "opposite" memory convention
-		//........................................................................
-		f1 = distodd[n];
-		f3 = distodd[N+n];
-		f5 = distodd[2*N+n];
-		f7 = distodd[3*N+n];
-		f9 = distodd[4*N+n];
-		f11 = distodd[5*N+n];
-		f13 = distodd[6*N+n];
-		f15 = distodd[7*N+n];
-		f17 = distodd[8*N+n];
-		//........................................................................
-		f0 = disteven[n];
-		f2 = disteven[N+n];
-		f4 = disteven[2*N+n];
-		f6 = disteven[3*N+n];
-		f8 = disteven[4*N+n];
-		f10 = disteven[5*N+n];
-		f12 = disteven[6*N+n];
-		f14 = disteven[7*N+n];
-		f16 = disteven[8*N+n];
-		f18 = disteven[9*N+n];
-		//........Determine the outlet flow velocity.........
-		//			uz = 1 - (f0+f3+f4+f1+f2+f7+f8+f10+f9+
-		//					  2*(f6+f16+f17+f12+f13))/dout;
-		//...................................................
-		//........Set the Unknown Distributions..............
-		//			f5 = f6 + 0.33333333333333338*dout*uz;
-		//			f15 = f16 + 0.16666666666666678*dout*uz;
-		//			f18 = f15+f3-f4-f16+f17+f7-f8+f10-f9;
-		//			f11= 0.5*(dout*uz+f6+ f16+f17+f12+f13-f5
-		//				  -f15-f18-f1+f2-f13+f12-f7+f8+f10-f9);
-		//			f14= dout*uz+f6+ f16+f17+f12+f13-f5-f15-f18-f11;
-		
-		uz = -1.0 + (f0+f4+f3+f2+f1+f8+f7+f9+f10 + 2*(f6+f16+f17+f12+f13))/dout;
-		
-		f5 = f6 - 0.33333333333333338*dout* uz;
-		f15 = f16 - 0.16666666666666678*dout* uz;
-		f18 = f15 - f4 + f3-f16+f17-f8+f7-f9+f10;
-		f11 = (-dout*uz+f6+ f16+f17+f12+f13-f5-f15-f18+f2-f1-f13+f12+f8-f7-f9+f10)*0.5;
-		f14 = -dout*uz+f6+ f16+f17+f12+f13-f5-f15-f18-f11;
-		//........Store in "opposite" memory location..........
-		distodd[2*N+n] = f5;
-		distodd[5*N+n] = f11;
-		disteven[7*N+n] = f14;
-		distodd[7*N+n] = f15;
-		disteven[9*N+n] = f18;
-		//...................................................
-		
-	}
-}
-//*************************************************************************
-inline void ComputeColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz)
-{
-	int n,N,i,j,k,nn;
-	// distributions
-	double f1,f2,f3,f4,f5,f6,f7,f8,f9;
-	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
-	double nx,ny,nz;
-	
-	// non-conserved moments
-	// additional variables needed for computations
-	
-	N = Nx*Ny*Nz;
-	
-	for ( n=0; n<N; n++){
-		//.......Back out the 3-D indices for node n..............
-		k = n/(Nx*Ny);
-		j = (n-Nx*Ny*k)/Nx;
-		i = n-Nx*Ny*k-Nx*j;
-		//........................................................................
-		//........Get 1-D index for this thread....................
-		//		n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
-		//........................................................................
-		//					COMPUTE THE COLOR GRADIENT
-		//........................................................................
-		//.................Read Phase Indicator Values............................
-		//........................................................................
-		nn = n-1;							// neighbor index (get convention)
-		if (i-1<0)		nn += Nx;			// periodic BC along the x-boundary
-		f1 = phi[nn];						// get neighbor for phi - 1
-		//........................................................................
-		nn = n+1;							// neighbor index (get convention)
-		if (!(i+1<Nx))	nn -= Nx;			// periodic BC along the x-boundary
-		f2 = phi[nn];						// get neighbor for phi - 2
-		//........................................................................
-		nn = n-Nx;							// neighbor index (get convention)
-		if (j-1<0)		nn += Nx*Ny;		// Perioidic BC along the y-boundary
-		f3 = phi[nn];					// get neighbor for phi - 3
-		//........................................................................
-		nn = n+Nx;							// neighbor index (get convention)
-		if (!(j+1<Ny))	nn -= Nx*Ny;		// Perioidic BC along the y-boundary
-		f4 = phi[nn];					// get neighbor for phi - 4
-		//........................................................................
-		nn = n-Nx*Ny;						// neighbor index (get convention)
-		if (k-1<0)		nn += Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-		f5 = phi[nn];					// get neighbor for phi - 5
-		//........................................................................
-		nn = n+Nx*Ny;						// neighbor index (get convention)
-		if (!(k+1<Nz))	nn -= Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-		f6 = phi[nn];					// get neighbor for phi - 6
-		//........................................................................
-		nn = n-Nx-1;						// neighbor index (get convention)
-		if (i-1<0)			nn += Nx;		// periodic BC along the x-boundary
-		if (j-1<0)			nn += Nx*Ny;	// Perioidic BC along the y-boundary
-		f7 = phi[nn];					// get neighbor for phi - 7
-		//........................................................................
-		nn = n+Nx+1;						// neighbor index (get convention)
-		if (!(i+1<Nx))		nn -= Nx;		// periodic BC along the x-boundary
-		if (!(j+1<Ny))		nn -= Nx*Ny;	// Perioidic BC along the y-boundary
-		f8 = phi[nn];					// get neighbor for phi - 8
-		//........................................................................
-		nn = n+Nx-1;						// neighbor index (get convention)
-		if (i-1<0)			nn += Nx;		// periodic BC along the x-boundary
-		if (!(j+1<Ny))		nn -= Nx*Ny;	// Perioidic BC along the y-boundary
-		f9 = phi[nn];					// get neighbor for phi - 9
-		//........................................................................
-		nn = n-Nx+1;						// neighbor index (get convention)
-		if (!(i+1<Nx))		nn -= Nx;		// periodic BC along the x-boundary
-		if (j-1<0)			nn += Nx*Ny;	// Perioidic BC along the y-boundary
-		f10 = phi[nn];					// get neighbor for phi - 10
-		//........................................................................
-		nn = n-Nx*Ny-1;						// neighbor index (get convention)
-		if (i-1<0)			nn += Nx;		// periodic BC along the x-boundary
-		if (k-1<0)			nn += Nx*Ny*Nz;	// Perioidic BC along the z-boundary
-		f11 = phi[nn];					// get neighbor for phi - 11
-		//........................................................................
-		nn = n+Nx*Ny+1;						// neighbor index (get convention)
-		if (!(i+1<Nx))		nn -= Nx;		// periodic BC along the x-boundary
-		if (!(k+1<Nz))	nn -= Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-		f12 = phi[nn];					// get neighbor for phi - 12
-		//........................................................................
-		nn = n+Nx*Ny-1;						// neighbor index (get convention)
-		if (i-1<0)			nn += Nx;		// periodic BC along the x-boundary
-		if (!(k+1<Nz))		nn -= Nx*Ny*Nz;	// Perioidic BC along the z-boundary
-		f13 = phi[nn];					// get neighbor for phi - 13
-		//........................................................................
-		nn = n-Nx*Ny+1;						// neighbor index (get convention)
-		if (!(i+1<Nx))		nn -= Nx;		// periodic BC along the x-boundary
-		if (k-1<0)			nn += Nx*Ny*Nz;	// Perioidic BC along the z-boundary
-		f14 = phi[nn];					// get neighbor for phi - 14
-		//........................................................................
-		nn = n-Nx*Ny-Nx;					// neighbor index (get convention)
-		if (j-1<0)		nn += Nx*Ny;		// Perioidic BC along the y-boundary
-		if (k-1<0)		nn += Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-		f15 = phi[nn];					// get neighbor for phi - 15
-		//........................................................................
-		nn = n+Nx*Ny+Nx;					// neighbor index (get convention)
-		if (!(j+1<Ny))	nn -= Nx*Ny;		// Perioidic BC along the y-boundary
-		if (!(k+1<Nz))	nn -= Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-		f16 = phi[nn];					// get neighbor for phi - 16
-		//........................................................................
-		nn = n+Nx*Ny-Nx;					// neighbor index (get convention)
-		if (j-1<0)		nn += Nx*Ny;		// Perioidic BC along the y-boundary
-		if (!(k+1<Nz))	nn -= Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-		f17 = phi[nn];					// get neighbor for phi - 17
-		//........................................................................
-		nn = n-Nx*Ny+Nx;					// neighbor index (get convention)
-		if (!(j+1<Ny))	nn -= Nx*Ny;		// Perioidic BC along the y-boundary
-		if (k-1<0)		nn += Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-		f18 = phi[nn];					// get neighbor for phi - 18
-		//............Compute the Color Gradient...................................
-		nx = -(f1-f2+0.5*(f7-f8+f9-f10+f11-f12+f13-f14));
-		ny = -(f3-f4+0.5*(f7-f8-f9+f10+f15-f16+f17-f18));
-		nz = -(f5-f6+0.5*(f11-f12-f13+f14+f15-f16-f17+f18));
-		//...........Normalize the Color Gradient.................................
-		//	C = sqrt(nx*nx+ny*ny+nz*nz);
-		//	nx = nx/C;
-		//	ny = ny/C;
-		//	nz = nz/C;
-		//...Store the Color Gradient....................
-		ColorGrad[3*n] = nx;
-		ColorGrad[3*n+1] = ny;
-		ColorGrad[3*n+2] = nz;
-		//...............................................
-	}
-}
-//*************************************************************************
-inline void ColorCollide( char *ID, double *disteven, double *distodd, double *ColorGrad,
-						 double *Velocity, int Nx, int Ny, int Nz, double rlx_setA, double rlx_setB,
-						 double alpha, double beta, double Fx, double Fy, double Fz, bool pBC)
-{
-	
-	int n,N;
-	// distributions
-	double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
-	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
-	
-	// non-conserved moments
-	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
-	// additional variables needed for computations
-	double rho,jx,jy,jz,C,nx,ny,nz;
-	
-	N = Nx*Ny*Nz;
-	char id;
-	
-	for ( n=0; n<N; n++){
-		id = ID[n];
-		
-		if (id > 0){
-			
-			// Retrieve the color gradient
-			nx = ColorGrad[3*n];
-			ny = ColorGrad[3*n+1];
-			nz = ColorGrad[3*n+2];
-			//...........Normalize the Color Gradient.................................
-			C = sqrt(nx*nx+ny*ny+nz*nz);
-			nx = nx/C;
-			ny = ny/C;
-			nz = nz/C;
-			//......No color gradient at z-boundary if pressure BC are set.............
-			//	if (pBC && k==0) nx = ny = nz = 0.f;
-			//	if (pBC && k==Nz-1) nx = ny = nz = 0.f;
-			//........................................................................
-			//					READ THE DISTRIBUTIONS
-			//		(read from opposite array due to previous swap operation)
-			//........................................................................
-			f2 = distodd[n];
-			f4 = distodd[N+n];
-			f6 = distodd[2*N+n];
-			f8 = distodd[3*N+n];
-			f10 = distodd[4*N+n];
-			f12 = distodd[5*N+n];
-			f14 = distodd[6*N+n];
-			f16 = distodd[7*N+n];
-			f18 = distodd[8*N+n];
-			//........................................................................
-			f0 = disteven[n];
-			f1 = disteven[N+n];
-			f3 = disteven[2*N+n];
-			f5 = disteven[3*N+n];
-			f7 = disteven[4*N+n];
-			f9 = disteven[5*N+n];
-			f11 = disteven[6*N+n];
-			f13 = disteven[7*N+n];
-			f15 = disteven[8*N+n];
-			f17 = disteven[9*N+n];
-			//........................................................................
-			//					PERFORM RELAXATION PROCESS
-			//........................................................................
-			//....................compute the moments...............................................
-			rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
-			m1 = -30*f0-11*(f2+f1+f4+f3+f6+f5)+8*(f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18 +f17);
-			m2 = 12*f0-4*(f2+f1 +f4+f3+f6 +f5)+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
-			jx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
-			m4 = 4*(-f1+f2)+f7-f8+f9-f10+f11-f12+f13-f14;
-			jy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
-			m6 = -4*(f3-f4)+f7-f8-f9+f10+f15-f16+f17-f18;
-			jz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
-			m8 = -4*(f5-f6)+f11-f12-f13+f14+f15-f16-f17+f18;
-			m9 = 2*(f1+f2)-f3-f4-f5-f6+f7+f8+f9+f10+f11+f12+f13+f14-2*(f15+f16+f17+f18);
-			m10 = -4*(f1+f2)+2*(f4+f3+f6+f5)+f8+f7+f10+f9+f12+f11+f14+f13-2*(f16+f15+f18+f17);
-			m11 = f4+f3-f6-f5+f8+f7+f10+f9-f12-f11-f14-f13;
-			m12 = -2*(f4+f3-f6-f5)+f8+f7+f10+f9-f12-f11-f14-f13;
-			m13 = f8+f7-f10-f9;
-			m14 = f16+f15-f18-f17;
-			m15 = f12+f11-f14-f13;
-			m16 = f7-f8+f9-f10-f11+f12-f13+f14;
-			m17 = -f7+f8+f9-f10+f15-f16+f17-f18;
-			m18 = f11-f12-f13+f14-f15+f16+f17-f18;
-			//..........Toelke, Fruediger et. al. 2006...............
-			if (C == 0.0)	nx = ny = nz = 1.0;
-			m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) -alpha*C - m1);
-			m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho)- m2);
-			m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4);
-			m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6);
-			m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8);
-			m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9);
-			m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
-			m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) + 0.5*alpha*C*(ny*ny-nz*nz)- m11);
-			m12 = m12 + rlx_setA*( -0.5*((jy*jy-jz*jz)/rho) - m12);
-			m13 = m13 + rlx_setA*( (jx*jy/rho) + 0.5*alpha*C*nx*ny - m13);
-			m14 = m14 + rlx_setA*( (jy*jz/rho) + 0.5*alpha*C*ny*nz - m14);
-			m15 = m15 + rlx_setA*( (jx*jz/rho) + 0.5*alpha*C*nx*nz - m15);
-			m16 = m16 + rlx_setB*( - m16);
-			m17 = m17 + rlx_setB*( - m17);
-			m18 = m18 + rlx_setB*( - m18);
-			//.................inverse transformation......................................................
-			f0 = 0.05263157894736842*rho-0.012531328320802*m1+0.04761904761904762*m2;
-			f1 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
-			+0.1*(jx-m4)+0.0555555555555555555555555*(m9-m10);
-			f2 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
-			+0.1*(m4-jx)+0.0555555555555555555555555*(m9-m10);
-			f3 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
-			+0.1*(jy-m6)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12);
-			f4 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
-			+0.1*(m6-jy)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12);
-			f5 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
-			+0.1*(jz-m8)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11);
-			f6 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
-			+0.1*(m8-jz)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11);
-			f7 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx+jy)+0.025*(m4+m6)
-			+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
-			+0.04166666666666666*m12+0.25*m13+0.125*(m16-m17);
-			f8 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2-0.1*(jx+jy)-0.025*(m4+m6)
-			+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
-			+0.04166666666666666*m12+0.25*m13+0.125*(m17-m16);
-			f9 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx-jy)+0.025*(m4-m6)
-			+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
-			+0.04166666666666666*m12-0.25*m13+0.125*(m16+m17);
-			f10 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jy-jx)+0.025*(m6-m4)
-			+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
-			+0.04166666666666666*m12-0.25*m13-0.125*(m16+m17);
-			f11 = 0.05263157894736842*rho+0.003341687552213868*m1
-			+0.003968253968253968*m2+0.1*(jx+jz)+0.025*(m4+m8)
-			+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
-			-0.04166666666666666*m12+0.25*m15+0.125*(m18-m16);
-			f12 = 0.05263157894736842*rho+0.003341687552213868*m1
-			+0.003968253968253968*m2-0.1*(jx+jz)-0.025*(m4+m8)
-			+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
-			-0.04166666666666666*m12+0.25*m15+0.125*(m16-m18);
-			f13 = 0.05263157894736842*rho+0.003341687552213868*m1
-			+0.003968253968253968*m2+0.1*(jx-jz)+0.025*(m4-m8)
-			+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
-			-0.04166666666666666*m12-0.25*m15-0.125*(m16+m18);
-			f14 = 0.05263157894736842*rho+0.003341687552213868*m1
-			+0.003968253968253968*m2+0.1*(jz-jx)+0.025*(m8-m4)
-			+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
-			-0.04166666666666666*m12-0.25*m15+0.125*(m16+m18);
-			f15 = 0.05263157894736842*rho+0.003341687552213868*m1
-			+0.003968253968253968*m2+0.1*(jy+jz)+0.025*(m6+m8)
-			-0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m17-m18);
-			f16 =  0.05263157894736842*rho+0.003341687552213868*m1
-			+0.003968253968253968*m2-0.1*(jy+jz)-0.025*(m6+m8)
-			-0.0555555555555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m18-m17);
-			f17 = 0.05263157894736842*rho+0.003341687552213868*m1
-			+0.003968253968253968*m2+0.1*(jy-jz)+0.025*(m6-m8)
-			-0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14+0.125*(m17+m18);
-			f18 = 0.05263157894736842*rho+0.003341687552213868*m1
-			+0.003968253968253968*m2+0.1*(jz-jy)+0.025*(m8-m6)
-			-0.0555555555555555555555555*m9-0.02777777777777778*m10-0.25*m14-0.125*(m17+m18);
-			//.......................................................................................................
-			// incorporate external force
-			f1 += 0.16666666*Fx;
-			f2 -= 0.16666666*Fx;
-			f3 += 0.16666666*Fy;
-			f4 -= 0.16666666*Fy;
-			f5 += 0.16666666*Fz;
-			f6 -= 0.16666666*Fz;
-			f7 += 0.08333333333*(Fx+Fy);
-			f8 -= 0.08333333333*(Fx+Fy);
-			f9 += 0.08333333333*(Fx-Fy);
-			f10 -= 0.08333333333*(Fx-Fy);
-			f11 += 0.08333333333*(Fx+Fz);
-			f12 -= 0.08333333333*(Fx+Fz);
-			f13 += 0.08333333333*(Fx-Fz);
-			f14 -= 0.08333333333*(Fx-Fz);
-			f15 += 0.08333333333*(Fy+Fz);
-			f16 -= 0.08333333333*(Fy+Fz);
-			f17 += 0.08333333333*(Fy-Fz);
-			f18 -= 0.08333333333*(Fy-Fz);
-			//*********** WRITE UPDATED VALUES TO MEMORY ******************
-			// Write the updated distributions
-			//....EVEN.....................................
-			disteven[n] = f0;
-			disteven[N+n] = f2;
-			disteven[2*N+n] = f4;
-			disteven[3*N+n] = f6;
-			disteven[4*N+n] = f8;
-			disteven[5*N+n] = f10;
-			disteven[6*N+n] = f12;
-			disteven[7*N+n] = f14;
-			disteven[8*N+n] = f16;
-			disteven[9*N+n] = f18;
-			//....ODD......................................
-			distodd[n] = f1;
-			distodd[N+n] = f3;
-			distodd[2*N+n] = f5;
-			distodd[3*N+n] = f7;
-			distodd[4*N+n] = f9;
-			distodd[5*N+n] = f11;
-			distodd[6*N+n] = f13;
-			distodd[7*N+n] = f15;
-			distodd[8*N+n] = f17;
-			//...Store the Velocity..........................
-			Velocity[3*n] = jx;
-			Velocity[3*n+1] = jy;
-			Velocity[3*n+2] = jz;
-			/*				//...Store the Color Gradient....................
-			 ColorGrad[3*n] = C*nx;
-			 ColorGrad[3*n+1] = C*ny;
-			 ColorGrad[3*n+2] = C*nz;
-			 */				//...............................................
-			//***************************************************************
-		}	// check if n is in the solid
-	} // check if n is in the domain
-}
-//*************************************************************************
-inline void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity,
-							  double beta, int Nx, int Ny, int Nz, bool pBC)
-{
-	char id;
-	
-	int idx;
-	int in,jn,kn,n,nn,N;
-	int q,Cqx,Cqy,Cqz;
-	//	int sendLoc;
-	
-	double na,nb;		// density values
-	double ux,uy,uz;	// flow velocity
-	double nx,ny,nz,C;	// color gradient components
-	double a1,a2,b1,b2;
-	double sp,delta;
-	double feq[6];		// equilibrium distributions
-	// Set of Discrete velocities for the D3Q19 Model
-	int D3Q7[3][3]={{1,0,0},{0,1,0},{0,0,1}};
-	N = Nx*Ny*Nz;
-	
-	for ( n=0; n<N; n++){
-		id = ID[n];
-		// Local Density Values
-		na = Copy[2*n];
-		nb = Copy[2*n+1];
-		if (id > 0 && na+nb > 0.0){
-			//.......Back out the 3-D indices for node n..............
-			int	k = n/(Nx*Ny);
-			int j = (n-Nx*Ny*k)/Nx;
-			int i = n-Nx*Ny*k-Nx*j;
-			//.....Load the Color gradient.........
-			nx = ColorGrad[3*n];
-			ny = ColorGrad[3*n+1];
-			nz = ColorGrad[3*n+2];
-			C = sqrt(nx*nx+ny*ny+nz*nz);
-			nx = nx/C;
-			ny = ny/C;
-			nz = nz/C;
-			//....Load the flow velocity...........
-			ux = Velocity[3*n];
-			uy = Velocity[3*n+1];
-			uz = Velocity[3*n+2];
-			//....Instantiate the density distributions
-			// Generate Equilibrium Distributions and stream
-			// Stationary value - distribution 0
-			Den[2*n] += 0.3333333333333333*na;
-			Den[2*n+1] += 0.3333333333333333*nb;
-			// Non-Stationary equilibrium distributions
-			feq[0] = 0.1111111111111111*(1+3*ux);
-			feq[1] = 0.1111111111111111*(1-3*ux);
-			feq[2] = 0.1111111111111111*(1+3*uy);
-			feq[3] = 0.1111111111111111*(1-3*uy);
-			feq[4] = 0.1111111111111111*(1+3*uz);
-			feq[5] = 0.1111111111111111*(1-3*uz);
-			// Construction and streaming for the components
-			for (idx=0; idx<3; idx++){
-				// Distribution index
-				q = 2*idx;
-				// Associated discrete velocity
-				Cqx = D3Q7[idx][0];
-				Cqy = D3Q7[idx][1];
-				Cqz = D3Q7[idx][2];
-				// Generate the Equilibrium Distribution
-				a1 = na*feq[q];
-				b1 = nb*feq[q];
-				a2 = na*feq[q+1];
-				b2 = nb*feq[q+1];
-				// Recolor the distributions
-				if (C > 0.0){
-					sp = nx*double(Cqx)+ny*double(Cqy)+nz*double(Cqz);
-					//if (idx > 2)	sp = 0.7071067811865475*sp;
-					//delta = sp*min( min(a1,a2), min(b1,b2) );
-					delta = na*nb/(na+nb)*0.1111111111111111*sp;
-					//if (a1>0 && b1>0){
-					a1 += beta*delta;
-					a2 -= beta*delta;
-					b1 -= beta*delta;
-					b2 += beta*delta;
-				}
-				
-				// .......Get the neighbor node..............
-				//nn = n + Stride[idx];
-				in = i+Cqx;
-				jn = j+Cqy;
-				kn = k+Cqz;
-				
-				// Adjust for periodic BC, if necessary
-				if (in<0) in+= Nx;
-				if (jn<0) jn+= Ny;
-				if (kn<0) kn+= Nz;
-				if (!(in<Nx)) in-= Nx;
-				if (!(jn<Ny)) jn-= Ny;
-				if (!(kn<Nz)) kn-= Nz;
-				// Perform streaming or bounce-back as needed
-				id = ID[kn*Nx*Ny+jn*Nx+in];
-				if (id == 0){							//.....Bounce-back Rule...........
-					Den[2*n] += a1;
-					Den[2*n+1] += b1;
-					//						atomicAdd(&Den[2*n], a1);
-					//						atomicAdd(&Den[2*n+1], b1);
-				}
-				else{
-					//......Push the "distribution" to neighboring node...........
-					// Index of the neighbor in the local process
-					//nn = (kn-zmin[rank]+1)*Nxp*Nyp + (jn-ymin[rank]+1)*Nxp + (in-xmin[rank]+1);
-					nn = kn*Nx*Ny+jn*Nx+in;
-					// Push to neighboring node
-					Den[2*nn] += a1;
-					Den[2*nn+1] += b1;
-					//						atomicAdd(&Den[2*nn], a1);
-					//						atomicAdd(&Den[2*nn+1], b1);
-				}
-				
-				// .......Get the neighbor node..............
-				q = 2*idx+1;
-				in = i-Cqx;
-				jn = j-Cqy;
-				kn = k-Cqz;
-				// Adjust for periodic BC, if necessary
-				if (in<0) in+= Nx;
-				if (jn<0) jn+= Ny;
-				if (kn<0) kn+= Nz;
-				if (!(in<Nx)) in-= Nx;
-				if (!(jn<Ny)) jn-= Ny;
-				if (!(kn<Nz)) kn-= Nz;
-				// Perform streaming or bounce-back as needed
-				id = ID[kn*Nx*Ny+jn*Nx+in];
-				if (id == 0){
-					//.....Bounce-back Rule...........
-					Den[2*n] += a2;
-					Den[2*n+1] += b2;
-					//					atomicAdd(&Den[2*n], a2);
-					//					atomicAdd(&Den[2*n+1], b2);
-				}
-				else{
-					//......Push the "distribution" to neighboring node...........
-					// Index of the neighbor in the local process
-					//nn = (kn-zmin[rank]+1)*Nxp*Nyp + (jn-ymin[rank]+1)*Nxp + (in-xmin[rank]+1);
-					nn = kn*Nx*Ny+jn*Nx+in;
-					// Push to neighboring node
-					Den[2*nn] += a2;
-					Den[2*nn+1] += b2;
-					//					atomicAdd(&Den[2*nn], a2);
-					//					atomicAdd(&Den[2*nn+1], b2);
-				}
-			}
-		}
-	}
-}
-
-inline void ComputePhi(char *ID, double *Phi, double *Copy, double *Den, int N)
-{
-	int n;
-	double Na,Nb;
-	//...................................................................
-	// Update Phi
-	for ( n=0; n<N; n++){
-		if (ID[n] > 0){
-			// Get the density value (Streaming already performed)
-			Na = Den[2*n];
-			Nb = Den[2*n+1];
-			Phi[n] = (Na-Nb)/(Na+Nb);
-			// Store the copy of the current density
-			Copy[2*n] = Na;
-			Copy[2*n+1] = Nb;
-			// Zero the Density value to get ready for the next streaming
-			Den[2*n] = 0.0;
-			Den[2*n+1] = 0.0;
-		}
-	}
-	//...................................................................
-}
@@ -1,21 +0,0 @@
-extern void InitDenColor(char *ID, double *Den, double *Phi, double das, double dbs, int N);
-extern void InitD3Q19(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz);
-
-extern void Compute_VELOCITY(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz);
-
-//*************************************************************************
-//*************************************************************************
-extern  void PressureBC_inlet(double *disteven, double *distodd, double din,
-			      int Nx, int Ny, int Nz);
-extern  void PressureBC_outlet(double *disteven, double *distodd, double dout,
-			       int Nx, int Ny, int Nz, int S, int outlet);
-//*************************************************************************
-extern void ComputeColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz);
-//*************************************************************************
-extern void ColorCollide( char *ID, double *disteven, double *distodd, double *ColorGrad,
-						 double *Velocity, int Nx, int Ny, int Nz, int S,double rlx_setA, double rlx_setB,
-			  double alpha, double beta, double Fx, double Fy, double Fz, bool pBC);
-//*************************************************************************
-extern void DensityStreamD3Q7(char *ID, double *Den, double *Copy, double *Phi, double *ColorGrad, double *Velocity,
-			      double beta, int Nx, int Ny, int Nz, bool pBC);
-extern void ComputePhi(char *ID, double *Phi, double *Copy, double *Den, int N);
@@ -1,189 +0,0 @@
-
-inline void PackDist(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){
-	//....................................................................................
-	// Pack distribution q into the send buffer for the listed lattice sites
-	// dist may be even or odd distributions stored by stream layout
-	//....................................................................................
-	int idx,n;
-	for (idx=0; idx<count; idx++){
-		n = list[idx];
-		sendbuf[start+idx] = dist[q*N+n];
-	}
-}
-
-inline void MapRecvDist(int q, int Cqx, int Cqy, int Cqz, int *list,  int start, int count,
-					   double *recvbuf, double *dist, int Nx, int Ny, int Nz){
-	//....................................................................................
-	// Unack distribution from the recv buffer
-	// Distribution q matche Cqx, Cqy, Cqz
-	// swap rule means that the distributions in recvbuf are OPPOSITE of q
-	// dist may be even or odd distributions stored by stream layout
-	//....................................................................................
-	int i,j,k,n,nn,idx;
-	int N = Nx*Ny*Nz;
-	for (idx=0; idx<count; idx++){
-		// Get the value from the list -- note that n is the index is from the send (non-local) process
-		n = list[idx];
-		// Get the 3-D indices
-		k = n/(Nx*Ny);
-		j = (n-Nx*Ny*k)/Nx;
-		i = n-Nx*Ny*k-Nz*j;
-		// Streaming for the non-local distribution
-		i += Cqx;
-		j += Cqy;
-		k += Cqz;
-/*		if (i < 0) i += Nx;
-		if (j < 0) j += Ny;
-		if (k < 0) k += Nz;
-		if (!(i<Nx)) i -= Nx;
-		if (!(j<Ny)) j -= Ny;
-		if (!(k<Nz)) k -= Nz;
-*/
-		nn = k*Nx*Ny+j*Nx+i;
-		// unpack the distribution to the proper location
-	//	if (recvbuf[start+idx] != dist[q*N+nn]){
-	//		printf("Stopping to check error \n");
-	//		printf("recvbuf[start+idx] = %f \n",recvbuf[start+idx]);
-	//		printf("dist[q*N+nn] = %f \n",dist[q*N+nn]);
-	//		printf("A bug! Again? \n");
-	//		idx = count;
-	//	}
-//		list[idx] = nn;
-		dist[q*N+nn] = recvbuf[start+idx];
-	}
-}
-
-//*************************************************************************
-inline void SwapD3Q19(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz)
-{
-	int n,nn,N;
-	// distributions
-	double f1,f2,f3,f4,f5,f6,f7,f8,f9;
-	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
-	
-	N = Nx*Ny*Nz;
-	
-	for (n=0; n<N; n++){
-		//.......Back out the 3-D indices for node n..............
-		int	k = n/(Nx*Ny);
-		int j = (n-Nx*Ny*k)/Nx;
-		int i = n-Nx*Ny*k-Nz*j;
-		
-		if (ID[n] > 0){
-			//........................................................................
-			// Retrieve even distributions from the local node (swap convention)
-			//		f0 = disteven[n];  // Does not particupate in streaming
-			f1 = distodd[n];
-			f3 = distodd[N+n];
-			f5 = distodd[2*N+n];
-			f7 = distodd[3*N+n];
-			f9 = distodd[4*N+n];
-			f11 = distodd[5*N+n];
-			f13 = distodd[6*N+n];
-			f15 = distodd[7*N+n];
-			f17 = distodd[8*N+n];
-			//........................................................................
-			
-			//........................................................................
-			// Retrieve odd distributions from neighboring nodes (swap convention)
-			//........................................................................
-			nn = n+1;							// neighbor index (pull convention)
-			if (!(i+1<Nx))	nn -= Nx;			// periodic BC along the x-boundary
-			//if (i+1<Nx){
-			f2 = disteven[N+nn];					// pull neighbor for distribution 2
-			if (f2 > 0){
-				distodd[n] = f2;
-				disteven[N+nn] = f1;
-			}
-			//}
-			//........................................................................
-			nn = n+Nx;							// neighbor index (pull convention)
-			if (!(j+1<Ny))	nn -= Nx*Ny;		// Perioidic BC along the y-boundary
-			//if (j+1<Ny){
-			f4 = disteven[2*N+nn];				// pull neighbor for distribution 4
-			if (f4 > 0){
-				distodd[N+n] = f4;
-				disteven[2*N+nn] = f3;
-				//	}
-			}
-			//........................................................................
-			nn = n+Nx*Ny;						// neighbor index (pull convention)
-			if (!(k+1<Nz))	nn -= Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-			//if (k+1<Nz){
-			f6 = disteven[3*N+nn];				// pull neighbor for distribution 6
-			if (f6 > 0){
-				distodd[2*N+n] = f6;
-				disteven[3*N+nn] = f5;
-				//	}
-			}
-			//........................................................................
-			nn = n+Nx+1;						// neighbor index (pull convention)
-			if (!(i+1<Nx))		nn -= Nx;		// periodic BC along the x-boundary
-			if (!(j+1<Ny))		nn -= Nx*Ny;	// Perioidic BC along the y-boundary
-			//if ((i+1<Nx) && (j+1<Ny)){
-			f8 = disteven[4*N+nn];				// pull neighbor for distribution 8
-			if (f8 > 0){
-				distodd[3*N+n] = f8;
-				disteven[4*N+nn] = f7;
-				//	}
-			}
-			//........................................................................
-			nn = n-Nx+1;						// neighbor index (pull convention)
-			if (!(i+1<Nx))	nn -= Nx;		// periodic BC along the x-boundary
-			if (j-1<0)		nn += Nx*Ny;	// Perioidic BC along the y-boundary
-			//if (!(i-1<0) && (j+1<Ny)){
-			f10 = disteven[5*N+nn];					// pull neighbor for distribution 9
-			if (f10 > 0){
-				distodd[4*N+n] = f10;
-				disteven[5*N+nn] = f9;
-				//	}
-			}
-			//........................................................................
-			nn = n+Nx*Ny+1;						// neighbor index (pull convention)
-			if (!(i+1<Nx))	nn -= Nx;		// periodic BC along the x-boundary
-			if (!(k+1<Nz))	nn -= Nx*Ny*Nz;	// Perioidic BC along the z-boundary
-			//if ( !(i-1<0) && !(k-1<0)){
-			f12 = disteven[6*N+nn];				// pull distribution 11
-			if (f12 > 0){
-				distodd[5*N+n] = f12;
-				disteven[6*N+nn] = f11;
-				//	}
-			}
-			//........................................................................
-			nn = n-Nx*Ny+1;						// neighbor index (pull convention)
-			if (!(i+1<Nx))	nn -= Nx;		// periodic BC along the x-boundary
-			if (k-1<0)		nn += Nx*Ny*Nz;	// Perioidic BC along the z-boundary
-			//if (!(i-1<0) && (k+1<Nz)){
-			f14 = disteven[7*N+nn];				// pull neighbor for distribution 13
-			if (f14 > 0){
-				distodd[6*N+n] = f14;
-				disteven[7*N+nn] = f13;
-				//	}
-			}
-			//........................................................................
-			nn = n+Nx*Ny+Nx;					// neighbor index (pull convention)
-			if (!(j+1<Ny))	nn -= Nx*Ny;		// Perioidic BC along the y-boundary
-			if (!(k+1<Nz))	nn -= Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-			//if (!(j-1<0) && !(k-1<0)){
-			f16 = disteven[8*N+nn];				// pull neighbor for distribution 15
-			if (f16 > 0){
-				distodd[7*N+n] = f16;
-				disteven[8*N+nn] = f15;
-				//	}
-			}
-			//........................................................................
-			nn = n-Nx*Ny+Nx;					// neighbor index (pull convention)
-			if (!(j+1<Ny))	nn -= Nx*Ny;		// Perioidic BC along the y-boundary
-			if (k-1<0)		nn += Nx*Ny*Nz;		// Perioidic BC along the z-boundary
-			//if (!(j-1<0) && (k+1<Nz)){
-			f18 = disteven[9*N+nn];				// pull neighbor for distribution 17
-			if (f18 > 0){
-				distodd[8*N+n] = f18;
-				disteven[9*N+nn] = f17;
-				//	}
-			}
-			//........................................................................
-			
-		}
-	}
-}
@@ -1,7 +0,0 @@
-
-extern void PackDist(int q, int *list, int start, int count, double *sendbuf, double *dist, int N);
-
-extern void MapRecvDist(int q, int Cqx, int Cqy, int Cqz, int *list,  int start, int count,
-			double *recvbuf, double *dist, int Nx, int Ny, int Nz);
-//*************************************************************************
-extern void SwapD3Q19(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz);
@@ -1,53 +0,0 @@
-// GPU Functions for D3Q7 Lattice Boltzmann Methods
-
-inline void PackValues(int *list, int count, double *sendbuf, double *Data, int N){
-	//....................................................................................
-	// Pack distribution q into the send buffer for the listed lattice sites
-	// dist may be even or odd distributions stored by stream layout
-	//....................................................................................
-	int idx,n;
-	for (idx=0; idx<count; idx++){
-		n = list[idx];
-		sendbuf[idx] = Data[n];
-	}
-}
-inline void UnpackValues(int *list, int count, double *recvbuf, double *Data, int N){
-	//....................................................................................
-	// Pack distribution q into the send buffer for the listed lattice sites
-	// dist may be even or odd distributions stored by stream layout
-	//....................................................................................
-	int idx,n;
-	for (idx=0; idx<count; idx++){
-		n = list[idx];
-		Data[n] = recvbuf[idx];
-	}
-}
-
-inline void PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){
-	//....................................................................................
-	// Pack distribution into the send buffer for the listed lattice sites
-	//....................................................................................
-	int idx,n,component;
-	for (idx=0; idx<count; idx++){
-		for (component=0; component<number; component++){
-			n = list[idx];
-			sendbuf[idx*number+component] = Data[number*n+component];
-		}
-	}
-}
-
-
-inline void UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){
-	//....................................................................................
-	// Unack distribution from the recv buffer
-	// Sum to the existing density value
-	//....................................................................................
-	int idx,n,component;
-	for (idx=0; idx<count; idx++){
-		for (component=0; component<number; component++){
-			n = list[idx];
-			Data[number*n+component] += recvbuf[idx*number+component];
-		}
-	}
-}
-
@@ -1,53 +0,0 @@
-// GPU Functions for D3Q7 Lattice Boltzmann Methods
-
-inline void PackValues(int *list, int count, double *sendbuf, double *Data, int N){
-	//....................................................................................
-	// Pack distribution q into the send buffer for the listed lattice sites
-	// dist may be even or odd distributions stored by stream layout
-	//....................................................................................
-	int idx,n;
-	for (idx=0; idx<count; idx++){
-		n = list[idx];
-		sendbuf[idx] = Data[n];
-	}
-}
-inline void UnpackValues(int *list, int count, double *recvbuf, double *Data, int N){
-	//....................................................................................
-	// Pack distribution q into the send buffer for the listed lattice sites
-	// dist may be even or odd distributions stored by stream layout
-	//....................................................................................
-	int idx,n;
-	for (idx=0; idx<count; idx++){
-		n = list[idx];
-		Data[n] = recvbuf[idx];
-	}
-}
-
-inline void PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){
-	//....................................................................................
-	// Pack distribution into the send buffer for the listed lattice sites
-	//....................................................................................
-	int idx,n,component;
-	for (idx=0; idx<count; idx++){
-		for (component=0; component<number; component++){
-			n = list[idx];
-			sendbuf[idx*number+component] = Data[number*n+component];
-		}
-	}
-}
-
-
-inline void UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){
-	//....................................................................................
-	// Unack distribution from the recv buffer
-	// Sum to the existing density value
-	//....................................................................................
-	int idx,n,component;
-	for (idx=0; idx<count; idx++){
-		for (component=0; component<number; component++){
-			n = list[idx];
-			Data[number*n+component] += recvbuf[idx*number+component];
-		}
-	}
-}
-
@@ -1,28 +0,0 @@
-CXX=$(VT_MPI_BIN)/mpicxx
-FLAGS=-O3
-
-ColorLBM-cpu:D3Q19.o D3Q7.o Color.o lb2_Color_mpi.o
-	$(CXX) $(FLAGS) -o ColorLBM-cpu lb2_Color_mpi.o D3Q19.o D3Q7.o Color.o
-
-D3Q19.o:D3Q19.cpp
-	$(CXX) $(FLAGS) -c -o D3Q19.o D3Q19.cpp
-
-D3Q7.o:D3Q7.cpp
-	$(CXX) $(FLAGS) -c -o D3Q7.o D3Q7.cpp
-
-Color.o:Color.cpp
-	$(CXX) $(FLAGS) -c -o Color.o Color.cpp
-
-lb2_Color_mpi.o:lb2_Color_mpi.cpp
-	$(CXX) $(FLAGS) -c -o lb2_Color_mpi.o lb2_Color_mpi.cpp
-
-#MRT-MPI.o:lb1_MRT_mpi.cpp
-#	$(CXX) -c -o MRT-MPI.o lb1_MRT_mpi.cpp
-
-#MRT-MPI:D3Q19.o MRT-MPI.o
-#	$(CXX) -o MRT-MPI D3Q19.o MRT-MPI.o
-
-clean:
-	rm *.o
-#	rm bin/*
-#	rm lib/*
@@ -0,0 +1,2 @@
+INSTALL_LBPM_EXE( lb1_MRT_mpi )
+INSTALL_LBPM_EXE( lb2_Color_mpi )
@@ -0,0 +1,56 @@
+\batchmode
+\documentclass[a4paper]{book}
+\usepackage{a4wide}
+\usepackage{makeidx}
+\usepackage{graphicx}
+\usepackage{multicol}
+\usepackage{float}
+\usepackage{listings}
+\usepackage{color}
+\usepackage{textcomp}
+\usepackage{alltt}
+\usepackage{times}
+\usepackage{ifpdf}
+\ifpdf
+\usepackage[pdftex,
+            pagebackref=true,
+            colorlinks=true,
+            linkcolor=blue,
+            unicode
+           ]{hyperref}
+\else
+\usepackage[ps2pdf,
+            pagebackref=true,
+            colorlinks=true,
+            linkcolor=blue,
+            unicode
+           ]{hyperref}
+\usepackage{pspicture}
+\fi
+\usepackage[utf8]{inputenc}
+\usepackage{doxygen}
+\lstset{language=C++,inputencoding=utf8,basicstyle=\footnotesize,breaklines=true,breakatwhitespace=true,tabsize=8,numbers=left }
+\makeindex
+\setcounter{tocdepth}{3}
+\renewcommand{\footrulewidth}{0.4pt}
+\begin{document}
+\hypersetup{pageanchor=false}
+\begin{titlepage}
+\vspace*{7cm}
+\begin{center}
+{\Huge Title}\\
+\vspace*{1cm}
+%%{\large Generated by Doxygen 1.6.3}\\
+\vspace*{0.5cm}
+%%{\small Tue Jan 24 16:37:18 2012}\\
+\end{center}
+\end{titlepage}
+\clearemptydoublepage
+\pagenumbering{roman}
+\tableofcontents
+\clearemptydoublepage
+\pagenumbering{arabic}
+\hypersetup{pageanchor=true}
+
+%% $title, $datetime, $date, $doxygenversion, $projectname, $projectnumber. Do
+
@@ -0,0 +1,23 @@
+<p>
+<hr size=4>
+<table cols=2 width=100% padding=2>
+  <tr>
+    <td align=left>
+    <small>
+        Title<br />
+    </small>
+  </td>
+  <td align=right>
+    <small>
+     This page automatically produced from the <br> source code by 
+     <a href="http://www.stack.nl/~dimitri/doxygen/index.html">
+     <img src="doxygen.png" alt="doxygen" align=center border=0 
+      width=59 height=26></a> <br>
+      Last updated: $datetime.  <br>
+      <a href="mailto:mail@server.org"> <i>Comments on this page</i></a>
+    </small>
+  </td>
+</tr>
+<hr size=4>
+</BODY>
+</HTML>
@@ -0,0 +1,18 @@
+<HTML>
+<HEAD>
+   <TITLE> $title </TITLE>
+   <LINK HREF="doxygen.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="tabs.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="draco.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="../../doxygen.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="../../tabs.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="../../draco.css" REL="stylesheet" TYPE="text/css" >
+</HEAD>
+<BODY BGCOLOR="#FFFFFF">
+
+<div class="header">
+   <br>
+   On-Line Documentation 
+</div>
+
+
@@ -0,0 +1,6 @@
+#INSTALL_LBPM_EXE( lb1_MRT_mpi )
+#INSTALL_LBPM_EXE( lb2_Color )
+#INSTALL_LBPM_EXE( lb2_Color_mpi )
+#INSTALL_LBPM_EXE( lb2_Color_pBC_wia_mpi )
+INSTALL_LBPM_EXE( lb2_Color_wia_mpi )
+
@@ -5,7 +5,11 @@
 #include <stdio.h>
 #include <iostream>
 #include <fstream>
+#include <string.h>
 #include <mpi.h>
+#include <stdlib.h>
+
+using namespace std;

 //*************************************************************************
 // MRT implementation of the LBM using CUDA
@@ -1,3 +1,4 @@
+#if 0
 #include "Array.h"
 /*
 *  Array.cpp
@@ -254,3 +255,4 @@ DoubleArray DoubleArray::Copy()
    return CopyInto;
 }

+#endif
@@ -0,0 +1,11 @@
+# Copy files for the tests
+CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cylindertest ${CMAKE_CURRENT_BINARY_DIR}/cylindertest COPYONLY )
+
+
+# Add the tests
+ADD_LBPM_TEST( pmmc_cylinder )
+ADD_LBPM_TEST( TestCylinderAreas )
+ADD_LBPM_TEST( TestSphereCurvature )
+
+
+