From 1124b82f05209d3ff2f650c9cf42ce82d0dab27f Mon Sep 17 00:00:00 2001
From: Markus Blatt <markus@dr-blatt.de>
Date: Wed, 9 Oct 2019 14:55:20 +0200
Subject: [PATCH] Move translation of global cartesian index of global grid to
 IoRank.

Previously, it was still assumed that all ranks knew the global grid
and each map on CollectDataToIORank::indexMaps_ was a mapping of
send/receive index to the index of the cell using the mapper of  the
corresponding global grid.

With this patch inside of CollectDataToIORank::DistributeIndexMapping
indexMaps is a mapping from send/receive index to global cartesian
index until the destructor is run. Inside of the destructor of the
iorank the remapping to mapped index of the global grid happens and
the ranks array is computed.
---
 ebos/collecttoiorank.hh | 60 ++++++++++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 12 deletions(-)

diff --git a/ebos/collecttoiorank.hh b/ebos/collecttoiorank.hh
index 8c817bd6a..0458d9fdd 100644
--- a/ebos/collecttoiorank.hh
+++ b/ebos/collecttoiorank.hh
@@ -101,6 +101,7 @@ public:
         IndexMapType& localIndexMap_;
         IndexMapStorageType& indexMaps_;
         std::map<int, int> globalPosition_;
+        std::set<int>& recv_;
         std::vector<int>& ranks_;
 
     public:
@@ -108,21 +109,26 @@ public:
                                const std::vector<int>& distributedGlobalIndex,
                                IndexMapType& localIndexMap,
                                IndexMapStorageType& indexMaps,
-                               std::vector<int>& ranks)
+                               std::vector<int>& ranks,
+                               std::set<int>& recv,
+                               bool isIORank)
             : distributedGlobalIndex_(distributedGlobalIndex)
             , localIndexMap_(localIndexMap)
             , indexMaps_(indexMaps)
             , globalPosition_()
+            , recv_(recv)
             , ranks_(ranks)
         {
             size_t size = globalIndex.size();
             // create mapping globalIndex --> localIndex
-            for (size_t index = 0; index < size; ++index)
-                globalPosition_.insert(std::make_pair(globalIndex[index], index));
+            if ( isIORank ) // ioRank
+                for (size_t index = 0; index < size; ++index)
+                    globalPosition_.insert(std::make_pair(globalIndex[index], index));
 
             // we need to create a mapping from local to global
             if (!indexMaps_.empty()) {
-                ranks_.resize(size, -1);
+                if (isIORank)
+                    ranks_.resize(size, -1);
                 // for the ioRank create a localIndex to index in global state map
                 IndexMapType& indexMap = indexMaps_.back();
                 size_t localSize = localIndexMap_.size();
@@ -130,12 +136,44 @@ public:
                 for (size_t i=0; i<localSize; ++i)
                 {
                     int id = distributedGlobalIndex_[localIndexMap_[i]];
-                    indexMap[i] = globalPosition_[id];
-                    ranks_[indexMap[i]] = ioRank;
+                    indexMap[i] = id;
                 }
             }
         }
 
+        ~DistributeIndexMapping()
+        {
+            if (!indexMaps_.size())
+                return;
+
+            if ( ranks_.size() )
+            {
+                auto rankIt = recv_.begin();
+                // translate index maps from global cartesian to index
+                for (auto& indexMap: indexMaps_)
+                {
+                    int rank = 0;
+                    if (rankIt != recv_.end())
+                        rank = *rankIt;
+
+                    for (auto&& entry: indexMap)
+                    {
+                        auto candidate = globalPosition_.find(entry);
+                        assert(candidate != globalPosition_.end());
+                        entry = candidate->second;
+                        // Using max should be backwards compatible
+                        ranks_[entry] = std::max(ranks_[entry], rank);
+                    }
+                    if (rankIt != recv_.end())
+                        ++rankIt;
+                }
+#ifndef NDEBUG
+                for (const auto& rank: ranks_)
+                    assert(rank>=0);
+#endif
+            }
+        }
+
         void pack(int link, MessageBufferType& buffer)
         {
             // we should only get one link
@@ -163,11 +201,7 @@ public:
             buffer.read(numCells);
             indexMap.resize(numCells);
             for (int index = 0; index < numCells; ++index) {
-                int globalId = -1;
-                buffer.read(globalId);
-                assert(globalPosition_.find(globalId) != globalPosition_.end());
-                indexMap[index] = globalPosition_[globalId];
-                ranks_[indexMap[index]] = link + 1;
+                buffer.read(indexMap[index]);
             }
         }
     };
@@ -270,7 +304,9 @@ public:
                                                     distributedCartesianIndex,
                                                     localIndexMap_,
                                                     indexMaps_,
-                                                    globalRanks_);
+                                                    globalRanks_,
+                                                    recv,
+                                                    isIORank());
             toIORankComm_.exchange(distIndexMapping);
         }
     }