GlobalGPUMgr.C

Go to the documentation of this file.

#include "CudaRecord.h"
#include "CudaUtils.h"
#include "GlobalGPUMgr.decl.h"
#include "GlobalGPUMgr.h"
#include "SynchronousCollectives.h"
#include "Node.h"
#include "SimParameters.h"
#include "NamdEventsProfiling.h"

#include "charm++.h"

#include <sched.h>
#include <pthread.h>

#include "DeviceCUDA.h"

#if defined(NAMD_CUDA) || defined(NAMD_HIP)
extern __thread DeviceCUDA *deviceCUDA;

RunWithAffinity::RunWithAffinity(const bool valid, const int tempCpu) {
  validCpu = valid && (tempCpu >= 0) && (tempCpu < CmiNumCores());

  if (validCpu) {
    current = CPU_ALLOC(CmiNumCores());
    temp    = CPU_ALLOC(CmiNumCores());
    size    = CPU_ALLOC_SIZE(CmiNumCores());

    CPU_ZERO_S(size, current);
    CPU_ZERO_S(size, temp);

    pthread_getaffinity_np(pthread_self(), size, current);

    CPU_SET_S(tempCpu, size, temp);
    pthread_setaffinity_np(pthread_self(), size, temp);
  }
}

RunWithAffinity::~RunWithAffinity() {
  if (validCpu) {
    pthread_setaffinity_np(pthread_self(), size, current);
  
    CPU_FREE(current);
    CPU_FREE(temp);
  }
}

GlobalGPUMgr::GlobalGPUMgr(void)
{
  if (CkpvAccess(GlobalGPUMgr_instance) == NULL) {
    CkpvAccess(GlobalGPUMgr_instance) = this;
  } else {
    NAMD_bug("GlobalGPUMgr instanced twice on same processor!");
  }
}

GlobalGPUMgr::~GlobalGPUMgr(void) { }

void GlobalGPUMgr::initialize() {
  SynchronousCollectives* syncColl = SynchronousCollectives::Object();

  isMasterPe = (CkMyPe() == deviceCUDA->getMasterPe());
  isMasterDevice = deviceCUDA->getIsMasterDevice();

  peToDeviceIDMap.resize(CkNumPes(), -1);
  peToDeviceIndexMap.resize(CkNumPes(), -1);
  syncColl->forceBarrierAll(); //Make sure maps are resized

  const int myDeviceId = (isMasterPe) ? deviceCUDA->getDeviceID() : -1;
  peToDeviceIDMap = syncColl->allGather<int>(myDeviceId, SynchronousCollectiveScope::all);

  // Count number of devices first
  numDevices = 0;
  for (int i = 0; i < CkNumPes(); i++) {
    if (peToDeviceIDMap[i] != -1) {
      numDevices++;
    }
  }

  deviceIndexToPeMap.resize(numDevices, 0);
 
  // Setup maps
  int current_device = 0;
  for (int i = 0; i < CkNumPes(); i++) {
    if (peToDeviceIDMap[i] != -1) {
      deviceIndexToPeMap[current_device] = i;
      peToDeviceIndexMap[i] = current_device;
      current_device++;
    }
  }

  deviceIndex = peToDeviceIndexMap[CkMyPe()];
}

void GlobalGPUMgr::initializeBackends() {
}

void GlobalGPUMgr::finalize() {
}

bool GlobalGPUMgr::getIsPmeDevice() {
  return deviceCUDA->getPmeDeviceIndex() == deviceIndex;
}

#endif  /* NAMD_CUDA || NAMD_HIP */

#include "GlobalGPUMgr.def.h"