PCPPCommUtil.C

Go to the documentation of this file.

#include "PCPPTypes.H"
#include "PCPPCommUtil.H"
#include "PCPPIntervalUtils.H"
#include "PCPPIO.H"

namespace pcpp {
  namespace comm {

    void ReduceTimers(double *processTimes,unsigned int nTimes,
                      std::vector<std::vector<double> > &parallelTimes,
                      pcpp::CommunicatorType &inCommunicator)
    {

      int numProcs = 1;
      MPI_Comm inComm(inCommunicator.GetCommunicator());
      MPI_Comm_size(inComm,&numProcs);

      parallelTimes.resize(3);
      parallelTimes[0].resize(nTimes);
      parallelTimes[1].resize(nTimes);
      parallelTimes[2].resize(nTimes);

      MPI_Reduce(processTimes,&(parallelTimes[0][0]),nTimes,MPI_DOUBLE,MPI_MIN,0,inComm);
      MPI_Reduce(processTimes,&(parallelTimes[1][0]),nTimes,MPI_DOUBLE,MPI_MAX,0,inComm);
      MPI_Reduce(processTimes,&(parallelTimes[2][0]),nTimes,MPI_DOUBLE,MPI_SUM,0,inComm);

      for(unsigned int iTime = 0;iTime < nTimes;iTime++){
        parallelTimes[2][iTime] /= numProcs;
      }

    }

    int SetupCartesianTopology(pcpp::CommunicatorType &parallelCommunicator,
                               pcpp::ParallelTopologyInfoType &topologyInfo,
                               pcpp::CommunicatorType &topoCommunicator,
                               std::ostream &messageStream)
    {
      int &numDimensions(topologyInfo.numDimensions);

      if(numDimensions <= 0){
        messageStream << "WARNING: No dimension configured for parallel topology, defaulting to 3.\n";
        numDimensions = 3;
      }

      std::ostringstream Ostr;
      Ostr << "Dimension of parallel topology: " << numDimensions << std::endl;
      messageStream << Ostr.str();
      pcpp::io::RenewStream(Ostr);

      std::vector<int> &dimDir(topologyInfo.dimDir);
      std::vector<int> &cartDecomp(topologyInfo.cartDecomposition);
      std::vector<int> &cartDirs(topologyInfo.cartDecompDirections);
      std::vector<int> &isPeriodic(topologyInfo.isPeriodic);
      std::vector<int> &topoCoords(topologyInfo.topoCoords);
      std::vector<int> &cartNeighbors(topologyInfo.neighborRanks);

      dimDir.resize(numDimensions,0);
      if(isPeriodic.size() != numDimensions)
        isPeriodic.resize(numDimensions,0);
      topoCoords.resize(numDimensions,0);

      int numNodes = parallelCommunicator.Size();
      Ostr << "Total number of processes in topology: " << numNodes << std::endl;
      messageStream << Ostr.str();
      pcpp::io::RenewStream(Ostr);

      if(!cartDecomp.empty()){

        dimDir = cartDecomp;
        messageStream << "Using user-specified decomp map.\n";

      } else if (!cartDirs.empty()){

        messageStream << "Using user-specified directional guidance for decomp.\n";
        int numCartDims = numDimensions;
        std::vector<int>::iterator cartDirIt = cartDirs.begin();
        std::vector<int>::iterator dimDirIt  = dimDir.begin();

        while(dimDirIt != dimDir.end() && cartDirIt != cartDirs.end()){

          if(*cartDirIt++ == 0){

            *dimDirIt = 1;
            numCartDims--;

          }
          dimDirIt++;
        }

        if(numCartDims < numDimensions){
          if(numCartDims == 1){
            dimDirIt = dimDir.begin();
            while(dimDirIt != dimDir.end()){
              if(*dimDirIt == 0)
                *dimDirIt = parallelCommunicator.Size();
              dimDirIt++;
            }
          } else if(numNodes > 1){

            // prime factorization of desired number of partitions
            std::deque<size_t> primeFactors(pcpp::util::PrimeFactors(numNodes));
            int numFac = primeFactors.size();
            
            // If the number of prime factors is larger than the 
            // number of possible dimenions, reduce by recursively
            // combining (i.e. multiplying) the two smallest factors.
            // Resulting deque is sorted smallest to largest
            while(numFac > numCartDims){
              primeFactors[1] *= primeFactors.front();
              primeFactors.pop_front();
              std::sort(primeFactors.begin(),primeFactors.end());
              numFac--;
            }
            dimDirIt = dimDir.begin();
            while(dimDirIt != dimDir.end() && !primeFactors.empty()){
              if(*dimDirIt == 0){
                if(!primeFactors.empty()){
                  *dimDirIt = primeFactors.front();
                  primeFactors.pop_front();
                } else {
                  *dimDirIt = 1;
                }
              } 
              dimDirIt++;            
            }  
          }
        }
      }
      
      if(parallelCommunicator.InitializeCartesianTopology(numNodes,numDimensions,dimDir,
                                                          isPeriodic,true,topoCommunicator)){
        messageStream << "Initialization of Cartesian process topology failed.\n";
        return(1);
      }
      topoCoords = topoCommunicator.CartCoordinates();
      topoCommunicator.CartNeighbors(cartNeighbors);
      topologyInfo.rank = topoCommunicator.Rank();

      // Report to user
      //       Ostr << "Parallel topology: (";
      //       std::vector<int>::iterator dimIterator = dimDir.begin();
      //       while(dimIterator != dimDir.end()){
      //         Ostr << *dimIterator++;
      //         if(dimIterator != dimDir.end())
      //           Ostr << ", ";
      //       }
      //       Ostr << ")" << std::endl
      //            << "Cart Rank: (" << topoCommunicator.Rank() << "/"
      //            << numNodes << "), Cart Coords:  (";
      //       pcpp::io::DumpContents(Ostr,topoCoords,",");
      //       Ostr << ")" << std::endl;
      //       Ostr << "Cart Neighbors: (";
      //       pcpp::io::DumpContents(Ostr,cartNeighbors,",");
      //       Ostr << ")" << std::endl;
      //       messageStream << Ostr.str();
      //       pcpp::io::RenewStream(Ostr);
      
      return(0);
    };

    bool CheckResult(bool &localResult,pcpp::CommunicatorType &testComm)
    {
      testComm.SetExit(0);
      if(!localResult){
        testComm.SetExit(1);
      }
      if(testComm.Check()){
        localResult = false;
        return(false);
      }
      testComm.SetExit(0);
      return(true);
    };
  };
};