LBPM/analysis/runAnalysis.h

/*
  Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
  Copyright Equnior ASA

  This file is part of the Open Porous Media project (OPM).
  OPM is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.
  OPM is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  You should have received a copy of the GNU General Public License
  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef RunAnalysis_H_INC
#define RunAnalysis_H_INC

#include "analysis/SubPhase.h"
#include "analysis/TwoPhase.h"
#include "analysis/analysis.h"
#include "common/Communication.h"
#include "common/ScaLBL.h"
#include "threadpool/thread_pool.h"
#include "models/ColorModel.h"
#include <limits.h>


// Types of analysis
enum class AnalysisType : uint64_t {
    AnalyzeNone        = 0,
    IdentifyBlobs      = 0x01,
    CopyPhaseIndicator = 0x02,
    CopySimState       = 0x04,
    ComputeAverages    = 0x08,
    CreateRestart      = 0x10,
    WriteVis           = 0x20,
    ComputeSubphase    = 0x40
};


//! Class to run the analysis in multiple threads
class runAnalysis
{
public:
    //! Constructor
    runAnalysis( std::shared_ptr<Database> db, const RankInfoStruct &rank_info,
        std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr<Domain> dm, int Np,
        bool Regular, IntArray Map );
    
    runAnalysis( ScaLBL_ColorModel &ColorModel);

    //! Destructor
    ~runAnalysis();

    //! Run the next analysis
    void run( int timestep, std::shared_ptr<Database> db, TwoPhase &Averages, const double *Phi,
        double *Pressure, double *Velocity, double *fq, double *Den );

    void basic( int timestep, std::shared_ptr<Database> db, SubPhase &Averages, const double *Phi,
        double *Pressure, double *Velocity, double *fq, double *Den );
    void WriteVisData( int timestep, std::shared_ptr<Database> vis_db, SubPhase &Averages,
        const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den );

    //! Finish all active analysis
    void finish();

    /*!
     *  \brief    Set the affinities
     *  \details  This function will create the analysis threads and set the affinity
     *      of this thread and all analysis threads.  If MPI_THREAD_MULTIPLE is not
     *      enabled, the analysis threads will be disabled and the analysis will run in the current
     * thread.
     * @param[in] method    Method used to control the affinities:
     *                      none - Don't use threads (runs all analysis in the current thread)
     *                      default - Create the specified number of threads, but don't load balance
     *                      independent - Create the necessary number of threads to fill all cpus,
     *                                and set the affinities based on the current process such
     *                                that all threads run on independent cores
     * @param[in] N_threads Number of threads, only used by some of the methods
     */
    void createThreads( const std::string &method = "default", int N_threads = 4 );


private:
    runAnalysis();

    // Determine the analysis to perform
    AnalysisType computeAnalysisType( int timestep );

public:
    class commWrapper
    {
    public:
        Utilities::MPI comm;
        int tag;
        runAnalysis *analysis;
        commWrapper( int tag, const Utilities::MPI &comm, runAnalysis *analysis );
        commWrapper()                         = delete;
        commWrapper( const commWrapper &rhs ) = delete;
        commWrapper &operator=( const commWrapper &rhs ) = delete;
        commWrapper( commWrapper &&rhs );
        ~commWrapper();
    };

    // Get a comm (not thread safe)
    commWrapper getComm();

private:
    std::array<int, 3> d_n; // Number of local cells
    std::array<int, 3> d_N; // Number of local cells with ghosts
    int d_Np;
    int d_rank;
    int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval;
    int d_subphase_analysis_interval;
    double d_beta;
    bool d_regular;
    std::string  format; // IO format string "silo" or "hdf5"
    
    ThreadPool d_tpool;
    RankInfoStruct d_rank_info;
    IntArray d_Map;
    std::shared_ptr<std::pair<int, IntArray>> d_last_ids;
    std::shared_ptr<std::pair<int, IntArray>> d_last_index;
    std::shared_ptr<std::vector<BlobIDType>> d_last_id_map;
    std::vector<IO::MeshDataStruct> d_meshData;
    std::string d_restartFile;
    Utilities::MPI d_comm;
    Utilities::MPI d_comms[1024];
    volatile bool d_comm_used[1024];
    std::shared_ptr<ScaLBL_Communicator> d_ScaLBL_Comm;

    // Ids of work items to use for dependencies
    ThreadPool::thread_id_t d_wait_blobID;
    ThreadPool::thread_id_t d_wait_analysis;
    ThreadPool::thread_id_t d_wait_subphase;
    ThreadPool::thread_id_t d_wait_vis;
    ThreadPool::thread_id_t d_wait_restart;

    // Friends
    friend commWrapper::~commWrapper();
};

#endif