Setup a distributed material law manager and copy values.

Since the refactoring to using opm-material a material law  manager for
the global grid was used. This meant that the properties used for
elements of the local grid were wrong. With this commit we set up a
manager that is based on the local grid only.

examples/flow.cpp

@@ -364,7 +364,7 @@ try
     // and initilialize new properties and states for it.
     if( mpi_size > 1 )
     {
-        Opm::distributeGridAndData( grid, eclipseState, state, new_props, geoprops, parallel_information, use_local_perm );
+        Opm::distributeGridAndData( grid, deck, eclipseState, state, new_props, geoprops, materialLawManager, parallel_information, use_local_perm );
     }
 
     // create output writer after grid is distributed, otherwise the parallel output

opm/autodiff/BlackoilPropsAdFromDeck.cpp

@@ -118,6 +118,7 @@ namespace Opm
 
 /// Constructor for properties on a subgrid
 BlackoilPropsAdFromDeck::BlackoilPropsAdFromDeck(const BlackoilPropsAdFromDeck& props,
+                                                 std::shared_ptr<MaterialLawManager> materialLawManager,
                                                  const int number_of_cells)
     : rock_(number_of_cells)
 {
@@ -129,7 +130,7 @@ BlackoilPropsAdFromDeck::BlackoilPropsAdFromDeck(const BlackoilPropsAdFromDeck&
         OPM_THROW(std::runtime_error, "The number of cells is has to be larger than 0.");
     }
 
-    materialLawManager_ = props.materialLawManager_;
+    materialLawManager_ = materialLawManager;
 
     // Copy properties that do not depend on the postion within the grid.
     satprops_         = props.satprops_;

opm/autodiff/BlackoilPropsAdFromDeck.hpp

@@ -138,8 +138,11 @@ namespace Opm
         /// the correct size but the values will be undefined.
         ///
         /// \param props            The property object to copy from.
-        /// \paramm number_of_cells The number of cells of the subgrid.
+        /// \param materialLawManager The container for the material law parameter objects.
+        ///                           Initialized only for the subgrid
+        /// \param number_of_cells  The number of cells of the subgrid.
         BlackoilPropsAdFromDeck(const BlackoilPropsAdFromDeck& props,
+                                std::shared_ptr<MaterialLawManager> materialLawManager,
                                 const int number_of_cells);
 
 

opm/autodiff/RedistributeDataHandles.hpp

@@ -34,10 +34,12 @@ namespace Opm
 
 template <class Grid>
 inline void distributeGridAndData( Grid& ,
+                                   Opm::DeckConstPtr deck ,
                                    EclipseStateConstPtr ,
                                    BlackoilState& ,
                                    BlackoilPropsAdFromDeck& ,
                                    DerivedGeology&,
+                                   std::shared_ptr<BlackoilPropsAdFromDeck::MaterialLawManager>&,
                                    boost::any& ,
                                    const bool )
 {
@@ -327,10 +329,12 @@ private:
 
 inline
 void distributeGridAndData( Dune::CpGrid& grid,
+                            Opm::DeckConstPtr deck,
                             EclipseStateConstPtr eclipseState,
                             BlackoilState& state,
                             BlackoilPropsAdFromDeck& properties,
                             DerivedGeology& geology,
+                            std::shared_ptr<BlackoilPropsAdFromDeck::MaterialLawManager>& material_law_manager,
                             boost::any& parallelInformation,
                             const bool useLocalPerm)
 {
@@ -342,8 +346,29 @@ void distributeGridAndData( Dune::CpGrid& grid,
     // distribute the grid and switch to the distributed view
     grid.loadBalance(eclipseState);
     grid.switchToDistributedView();
+    std::vector<int> compressedToCartesianIdx;
+    Opm::createGlobalCellArray(grid, compressedToCartesianIdx);
+    typedef BlackoilPropsAdFromDeck::MaterialLawManager MaterialLawManager;
+    auto distributed_material_law_manager = std::make_shared<MaterialLawManager>();
+    distributed_material_law_manager->initFromDeck(deck, eclipseState, compressedToCartesianIdx);
+    // copy the values from the global to the local MaterialLawManager
+    // We should actually communicate these to be future proof. But that is
+    // really, really cumbersome for the underlying vector<shared_ptr>
+    // where the classes pointed to even have more shared_ptr stored in them.
+    typedef Dune::CpGrid::ParallelIndexSet IndexSet;
+    typedef IndexSet::IndexPair Pair;
+    const IndexSet& local_indices  = grid.getCellIndexSet();
+    for(auto index : local_indices)
+    {
+        distributed_material_law_manager->materialLawParamsPointer(index.local()) =
+            material_law_manager->materialLawParamsPointer(index.global());
 
-    BlackoilPropsAdFromDeck distributed_props(properties, grid.numCells());
+        distributed_material_law_manager->oilWaterScaledEpsInfoDrainagePointer(index.local()) =
+            material_law_manager->oilWaterScaledEpsInfoDrainagePointer(index.global());
+    }
+    BlackoilPropsAdFromDeck distributed_props(properties,
+                                              distributed_material_law_manager,
+                                              grid.numCells());
     distributed_state.init(grid.numCells(), grid.numFaces(), state.numPhases());
     // init does not resize surfacevol. Do it manually.
     distributed_state.surfacevol().resize(grid.numCells()*state.numPhases(),
@@ -367,7 +392,7 @@ void distributeGridAndData( Dune::CpGrid& grid,
     properties           = distributed_props;
     geology              = distributed_geology;
     state                = distributed_state;
-
+    material_law_manager = distributed_material_law_manager;
     extractParallelGridInformationToISTL(grid, parallelInformation);
 }
 #endif