ReductionMgr Class Reference

#include <ReductionMgr.h>

Inheritance diagram for ReductionMgr:

Public Member Functions
	ReductionMgr ()
	~ReductionMgr ()
void	buildSpanTree (const int pe, const int max_intranode_children, const int max_internode_children, int *parent, int *num_children, int **children)
SubmitReduction *	willSubmit (int setID, int size=-1)
RequireReduction *	willRequire (int setID, int size=-1)
void	remoteRegister (ReductionRegisterMsg *msg)
void	remoteUnregister (ReductionRegisterMsg *msg)
void	remoteSubmit (ReductionSubmitMsg *msg)

Static Public Member Functions
static ReductionMgr *	Object (void)

Friends
class	SubmitReduction
class	SubmitReductionCharm
class	SubmitReductionShared
class	RequireReduction
class	RequireReductionCharm
class	RequireReductionShared

Detailed Description

Definition at line 241 of file ReductionMgr.h.

Constructor & Destructor Documentation

ReductionMgr()

ReductionMgr::ReductionMgr

(

)

Definition at line 282 of file ReductionMgr.C.

References buildSpanTree(), DebugM, REDUCTION_MAX_CHILDREN, and REDUCTION_MAX_SET_ID.

                           {
    if (CkpvAccess(ReductionMgr_instance) == 0) {
      CkpvAccess(ReductionMgr_instance) = this;
    } else {
      DebugM(1, "ReductionMgr::ReductionMgr() - another instance exists!\n");
    }
    
    buildSpanTree(CkMyPe(),REDUCTION_MAX_CHILDREN,REDUCTION_MAX_CHILDREN,
                  &myParent,&numChildren,&children);
    
//    CkPrintf("TREE [%d] parent %d %d children\n",
//      CkMyPe(),myParent,numChildren);
//    if (numChildren > 0) {
//      for(int i=0; i < numChildren; i++)  {
//        CkPrintf("TREE [%d] child %d %d\n",CkMyPe(),i,children[i]);
//      }
//    }
    
    // fill in the spanning tree fields
#if 0  // Old spanning tree
    if (CkMyPe() == 0) {
      myParent = -1;
    } else {
      myParent = (CkMyPe()-1)/REDUCTION_MAX_CHILDREN;
    }
    firstChild = CkMyPe()*REDUCTION_MAX_CHILDREN + 1;
    if (firstChild > CkNumPes()) firstChild = CkNumPes();
    lastChild = firstChild + REDUCTION_MAX_CHILDREN;
    if (lastChild > CkNumPes()) lastChild = CkNumPes();
#endif

    // initialize data
    for(int i=0; i<REDUCTION_MAX_SET_ID; i++) {
      reductionSets[i] = 0;
    }

    DebugM(1,"ReductionMgr() instantiated.\n");
}

~ReductionMgr()

ReductionMgr::~ReductionMgr

(

)

Definition at line 322 of file ReductionMgr.C.

References REDUCTION_MAX_SET_ID.

                            {
    if (children != 0)
      delete [] children;
    for(int i=0; i<REDUCTION_MAX_SET_ID; i++) {
      delete reductionSets[i];
    }

}

Member Function Documentation

buildSpanTree()

void ReductionMgr::buildSpanTree	(	const int	pe,
		const int	max_intranode_children,
		const int	max_internode_children,
		int *	parent,
		int *	num_children,
		int **	children
	)

Definition at line 122 of file ReductionMgr.C.

References NAMD_bug(), NAMD_die(), and split().

Referenced by ReductionMgr().

{
  // If pe is a first-node, children are same-node pes and perhaps some
  // other first-nodes, and parents are other first-nodes. If pe is not a 
  // first-node, build the spanning tree among the children, and the parent
  // is the corresponding first-node

  // No matter what, build list of PEs on my node first
  const int num_pes = CkNumPes();
  const int num_node_pes = CmiNumPesOnPhysicalNode(CmiPhysicalNodeID(pe)); 
  int *node_pes = new int[num_node_pes];
  int pe_index = -1;
  const int first_pe = CmiGetFirstPeOnPhysicalNode(CmiPhysicalNodeID(pe));
  int num_nodes = 0;
  int *node_ids = new int[num_pes];
  int first_pe_index = -1;
  int my_parent_index;
  
  // Make sure PE 0 is a first-node
  if (pe == 0 && first_pe != pe) {
    NAMD_die("PE 0 is not the first physical node. This shouldn't happen");
  }
  // Get all the PEs on my node, and also build the list of all first-nodes
  int i;
  int node_pe_count=0;
  for (i = 0; i < num_pes; i++) {
    // Save first-nodes
    if (CmiGetFirstPeOnPhysicalNode(CmiPhysicalNodeID(i)) == i) {
      node_ids[num_nodes] = i;
      if (i == first_pe)
        first_pe_index = num_nodes;
      num_nodes++;
    }

    // Also, find pes on my node
    const int i1 = (i + first_pe) % num_pes;
    if (CmiPeOnSamePhysicalNode(first_pe,i1)) {
      if ( node_pe_count == num_node_pes )
        NAMD_bug("ReductionMgr::buildSpanTree found inconsistent physical node data from Charm++ runtime");
      node_pes[node_pe_count] = i1;
      if (pe == i1)
        pe_index = node_pe_count;
      node_pe_count++;
    }
  }
  if ( pe_index < 0 || first_pe_index < 0 )
    NAMD_bug("ReductionMgr::buildSpanTree found inconsistent physical node data from Charm++ runtime");
  
  // Any PE might have children on the same node, plus, if its a first-node,
  // it may have several children on other nodes

  int first_loc_child_index = pe_index * max_intranode_children + 1;
  int last_loc_child_index 
    = first_loc_child_index + max_intranode_children - 1;
  if (first_loc_child_index > num_node_pes) {
    first_loc_child_index = num_node_pes;
    last_loc_child_index = num_node_pes;
  } else {
    if (last_loc_child_index >= num_node_pes) 
      last_loc_child_index = num_node_pes-1;
  }
//  CkPrintf("Local [%d] firstpe %d max %d num %d firstloc %d lastloc %d\n",
//           pe,pe_index,max_intranode_children,num_node_pes,
//           first_loc_child_index,last_loc_child_index);
  
  int first_rem_child_index = num_nodes;
  int last_rem_child_index = num_nodes;
  int rem_children=0;
  int *rem_child_index = new int[max_internode_children];
  
  if (first_pe != pe) {
    // I'm not a first_pe, so I have no more children, and my parent
    // is someone else on my node
    my_parent_index = (pe_index-1)/max_intranode_children;
    *parent = node_pes[my_parent_index];
  } else {
    // I am a first_pe, so I may have additional children
    // on other nodes, and my parent will be on another node

    int range_begin = 0;
    int range_end = num_nodes;

    if (pe == 0) {
      my_parent_index = -1;
      *parent = -1;
    } else {
      my_parent_index = 0;
      while ( first_pe_index != range_begin ) {
        my_parent_index = range_begin;
        ++range_begin;
        for ( int i = 0; i < max_internode_children; ++i ) {
          int split = range_begin + ( range_end - range_begin ) / ( max_internode_children - i );
          if ( first_pe_index < split ) { range_end = split; break; } 
          else { range_begin = split; }
        }
      }
      *parent = node_ids[my_parent_index];
    }

    // now we know parent and need only repeat calculation of children
    int prev_child_index = range_begin;
    ++range_begin;
    for ( int i = 0; i < max_internode_children; ++i ) {
      if ( range_begin >= range_end ) break;
      if ( range_begin > prev_child_index ) {
        rem_child_index[rem_children++] = prev_child_index = range_begin;
      }
      range_begin += ( range_end - range_begin ) / ( max_internode_children - i );
    }
  }

  *num_children = 0;
  //CkPrintf("TREE pe %d my_parent %d %d\n",pe,my_parent_index,*parent);

  int loc_children=0;
  if (first_loc_child_index != num_node_pes) {
    loc_children = last_loc_child_index - first_loc_child_index + 1;
    *num_children += loc_children;
//    CkPrintf("TREE pe %d %d local children\n",pe,loc_children);
//  } else {
//    CkPrintf("TREE pe %d No local children\n",pe);
  }

  if (rem_children) {
    *num_children += rem_children;
//    CkPrintf("TREE pe %d %d rem children\n",pe,rem_children);
//  } else {
//    CkPrintf("TREE pe %d No rem children\n",pe);
  }
  if (*num_children == 0)
    *children = 0;
  else {
    *children = new int[*num_children];
//    CkPrintf("TREE pe %d children %d\n",pe,*num_children);
    int k;
    int child=0;
    if (loc_children > 0) {
      for(k=first_loc_child_index; k <= last_loc_child_index; k++) {
//        CkPrintf("TREE pe %d loc child[%d,%d] %d\n",pe,child,k,node_pes[k]);
        (*children)[child++]=node_pes[k];
      }
    }
    if (rem_children > 0) {
      for(k=0; k < rem_children; k++)  {
//        CkPrintf("TREE pe %d rem child[%d,%d] %d\n",pe,child,k,node_ids[rem_child_index[k]]);
        (*children)[child++]=node_ids[rem_child_index[k]];
      }
    }
  }
  delete [] rem_child_index;
  delete [] node_ids;
  delete [] node_pes;
}

Object()

static ReductionMgr* ReductionMgr::Object ( void  )

inlinestatic

Definition at line 290 of file ReductionMgr.h.

Referenced by colvarproxy_namd::colvarproxy_namd(), ComputeConsForce::ComputeConsForce(), ComputeConsTorque::ComputeConsTorque(), ComputeCylindricalBC::ComputeCylindricalBC(), ComputeEField::ComputeEField(), ComputeEwald::ComputeEwald(), ComputeExt::ComputeExt(), ComputeFmmSerial::ComputeFmmSerial(), ComputeFullDirect::ComputeFullDirect(), ComputeGBISser::ComputeGBISser(), ComputeGlobal::ComputeGlobal(), ComputeGridForce::ComputeGridForce(), ComputeHomeTuples< TholeElem, Thole, TholeValue >::ComputeHomeTuples(), ComputeLCPO::ComputeLCPO(), ComputeLjPmeSerial::ComputeLjPmeSerial(), ComputeMsm::ComputeMsm(), ComputeMsmSerial::ComputeMsmSerial(), ComputeNonbondedPair::ComputeNonbondedPair(), ComputeNonbondedSelf::ComputeNonbondedSelf(), ComputeQM::ComputeQM(), ComputeRestraints::ComputeRestraints(), ComputeSphericalBC::ComputeSphericalBC(), ComputeStir::ComputeStir(), ComputeTclBC::ComputeTclBC(), Controller::Controller(), CudaPmeOneDevice::CudaPmeOneDevice(), GlobalMaster::GlobalMaster(), ComputePmeCUDA::initialize(), CudaComputeNonbonded::initialize(), ComputePmeMgr::initialize_computes(), Patch::Patch(), PmePencilXYZ::PmePencilXYZ(), PmePencilZ::PmePencilZ(), PressureProfileReduction::PressureProfileReduction(), and Sequencer::Sequencer().

                                           {
    return CkpvAccess(ReductionMgr_instance);
  }

remoteRegister()

void ReductionMgr::remoteRegister

(

ReductionRegisterMsg *

msg

)

Definition at line 426 of file ReductionMgr.C.

References ReductionRegisterMsg::dataSize, NAMD_die(), ReductionRegisterMsg::reductionSetID, and ReductionRegisterMsg::sourceNode.

                                                           {

  int setID = msg->reductionSetID;
  int size = msg->dataSize;
  ReductionSet *set = getSet(setID,size);
  if ( set->getData(set->nextSequenceNumber)->submitsRecorded ) {
    NAMD_die("ReductionMgr::remoteRegister called while reductions outstanding on parent!");
  }

  set->submitsRegistered++;
  set->addToRemoteSequenceNumber[childIndex(msg->sourceNode)]
                                        = set->nextSequenceNumber;
//  CkPrintf("[%d] reduction register received from node[%d] %d\n",
//    CkMyPe(),childIndex(msg->sourceNode),msg->sourceNode);
    
  delete msg;
}

remoteSubmit()

void ReductionMgr::remoteSubmit

(

ReductionSubmitMsg *

msg

)

Definition at line 460 of file ReductionMgr.C.

References ReductionSubmitMsg::data, ReductionSetData::data, ReductionSubmitMsg::dataSize, NAMD_bug(), REDUCTIONS_MINIMIZER, ReductionSubmitMsg::reductionSetID, ReductionSubmitMsg::sequenceNumber, ReductionSubmitMsg::sourceNode, and ReductionSetData::submitsRecorded.

                                                       {
  int setID = msg->reductionSetID;
  ReductionSet *set = reductionSets[setID];
  int seqNum = msg->sequenceNumber
        + set->addToRemoteSequenceNumber[childIndex(msg->sourceNode)];

//iout << "seq " << seqNum << " from " << msg->sourceNode << " received on " << CkMyPe() << "\n" << endi;
  int size = msg->dataSize;
  if ( size != set->dataSize ) {
    NAMD_bug("ReductionMgr::remoteSubmit data sizes do not match.");
  }

  BigReal *newData = msg->data;
  ReductionSetData *data = set->getData(seqNum);
  BigReal *curData = data->data;
#ifdef ARCH_POWERPC
#pragma disjoint (*curData,  *newData)
#pragma unroll(4)
#endif
  if ( setID == REDUCTIONS_MINIMIZER ) {
    for ( int i = 0; i < size; ++i ) {
      if ( newData[i] > curData[i] ) {
        curData[i] = newData[i];
      }
    }
  } else {
    for ( int i = 0; i < size; ++i ) {
      curData[i] += newData[i];
    }
  }
//  CkPrintf("[%d] reduction Submit received from node[%d] %d\n",
//    CkMyPe(),childIndex(msg->sourceNode),msg->sourceNode);
  delete msg;

  data->submitsRecorded++;
  if ( data->submitsRecorded == set->submitsRegistered ) {
    mergeAndDeliver(set,seqNum);
  }
}

remoteUnregister()

void ReductionMgr::remoteUnregister

(

ReductionRegisterMsg *

msg

)

Definition at line 445 of file ReductionMgr.C.

References NAMD_die(), and ReductionRegisterMsg::reductionSetID.

                                                             {

  int setID = msg->reductionSetID;
  ReductionSet *set = reductionSets[setID];
  if ( set->getData(set->nextSequenceNumber)->submitsRecorded ) {
    NAMD_die("SubmitReduction deleted while reductions outstanding on parent!");
  }

  set->submitsRegistered--;

  delSet(setID);
  delete msg;
}

willRequire()

RequireReduction * ReductionMgr::willRequire	(	int	setID,
		int	size = -1
	)

Definition at line 539 of file ReductionMgr.C.

References NAMD_die(), Node::Object(), REDUCTIONS_GPURESIDENT, RequireReductionCharm, RequireReductionShared, Node::simParameters, and simParams.

Referenced by Controller::Controller(), and PressureProfileReduction::PressureProfileReduction().

                                                               {
  SimParameters *simParams = Node::Object()->simParameters;
  RequireReduction *handle;
  // Only use the shared memory reductions for GPU-resident dynamics
  if (simParams->GPUresidentSingleProcessMode && setID == REDUCTIONS_GPURESIDENT) {
    handle = new RequireReductionShared;
  } else {
    ReductionSet *set = getSet(setID,size);
    set->requireRegistered++;
    if ( set->getData(set->nextSequenceNumber)->submitsRecorded ) {
      NAMD_die("ReductionMgr::willRequire called while reductions outstanding!");
    }

    RequireReductionCharm* charm_handle = new RequireReductionCharm;
    charm_handle->reductionSetID = setID;
    charm_handle->sequenceNumber = set->nextSequenceNumber;
    charm_handle->master = this;

    handle = (RequireReduction*) charm_handle;

  }
  return handle;
}

willSubmit()

SubmitReduction * ReductionMgr::willSubmit	(	int	setID,
		int	size = -1
	)

Definition at line 368 of file ReductionMgr.C.

References ReductionSetData::data, SubmitReduction::data, SubmitReductionCharm::master, NAMD_die(), Node::Object(), REDUCTIONS_GPURESIDENT, SubmitReductionCharm::reductionSetID, SubmitReductionCharm::sequenceNumber, Node::simParameters, simParams, SubmitReductionCharm, SubmitReductionShared, and ReductionSetData::submitsRecorded.

Referenced by colvarproxy_namd::colvarproxy_namd(), ComputeConsForce::ComputeConsForce(), ComputeConsTorque::ComputeConsTorque(), ComputeCylindricalBC::ComputeCylindricalBC(), ComputeEField::ComputeEField(), ComputeEwald::ComputeEwald(), ComputeExt::ComputeExt(), ComputeFmmSerial::ComputeFmmSerial(), ComputeFullDirect::ComputeFullDirect(), ComputeGBISser::ComputeGBISser(), ComputeGlobal::ComputeGlobal(), ComputeGridForce::ComputeGridForce(), ComputeHomeTuples< TholeElem, Thole, TholeValue >::ComputeHomeTuples(), ComputeLCPO::ComputeLCPO(), ComputeLjPmeSerial::ComputeLjPmeSerial(), ComputeMsm::ComputeMsm(), ComputeMsmSerial::ComputeMsmSerial(), ComputeNonbondedPair::ComputeNonbondedPair(), ComputeNonbondedSelf::ComputeNonbondedSelf(), ComputeQM::ComputeQM(), ComputeRestraints::ComputeRestraints(), ComputeSphericalBC::ComputeSphericalBC(), ComputeStir::ComputeStir(), ComputeTclBC::ComputeTclBC(), Controller::Controller(), CudaPmeOneDevice::CudaPmeOneDevice(), GlobalMaster::GlobalMaster(), ComputePmeCUDA::initialize(), CudaComputeNonbonded::initialize(), ComputePmeMgr::initialize_computes(), Patch::Patch(), PmePencilXYZ::PmePencilXYZ(), PmePencilZ::PmePencilZ(), and Sequencer::Sequencer().

                                                             {
  SimParameters *simParams = Node::Object()->simParameters;

  SubmitReduction *handle;
  // Only use the shared memory reductions for GPU-resident dynamics
  if (simParams->GPUresidentSingleProcessMode && setID == REDUCTIONS_GPURESIDENT) {
    handle = new SubmitReductionShared;
  } else {
    ReductionSet *set = getSet(setID, size);
    ReductionSetData *data = set->getData(set->nextSequenceNumber);
    if ( data->submitsRecorded ) {
      NAMD_die("ReductionMgr::willSubmit called while reductions outstanding!");
    }

    set->submitsRegistered++;

    SubmitReductionCharm* charm_handle = new SubmitReductionCharm;
    charm_handle->reductionSetID = setID;
    charm_handle->sequenceNumber = set->nextSequenceNumber;
    charm_handle->master = this;
    charm_handle->data = data->data;

    handle = (SubmitReduction*) charm_handle;
  }

  return handle;
}

Friends And Related Function Documentation

RequireReduction

friend class RequireReduction

friend

Definition at line 247 of file ReductionMgr.h.

RequireReductionCharm

friend class RequireReductionCharm

friend

Definition at line 248 of file ReductionMgr.h.

Referenced by willRequire().

RequireReductionShared

friend class RequireReductionShared

friend

Definition at line 249 of file ReductionMgr.h.

Referenced by willRequire().

SubmitReduction

friend class SubmitReduction

friend

Definition at line 244 of file ReductionMgr.h.

SubmitReductionCharm

friend class SubmitReductionCharm

friend

Definition at line 245 of file ReductionMgr.h.

Referenced by willSubmit().

SubmitReductionShared

friend class SubmitReductionShared

friend

Definition at line 246 of file ReductionMgr.h.

Referenced by willSubmit().

---

The documentation for this class was generated from the following files:

• ReductionMgr.h
• ReductionMgr.C