PlasCom2  1.0
XPACC Multi-physics simluation application
TestHalo.C
Go to the documentation of this file.
1 #include "Testing.H"
2 #include "Simulation.H"
3 #include "PCPPCommUtil.H"
4 #include "PCPPReport.H"
5 #include "PCPPIntervalUtils.H"
6 
7 using pcpp::comm::CheckResult;
8 
9 void TestHaloParallel(ix::test::results &parallelUnitResults,pcpp::CommunicatorType &testComm)
10 {
11  pcpp::CommunicatorType myComm(testComm);
12  pcpp::CommunicatorType myCartComm;
13 
14  std::ostringstream messageStream;
15  pcpp::ParallelTopologyInfoType cartInfo;
16  pcpp::comm::SetupCartesianTopology(myComm,cartInfo,myCartComm,messageStream);
17  std::vector<int> &cartCoords(myCartComm.CartCoordinates());
18  std::vector<int> &cartDims(myCartComm.CartDimensions());
19  std::vector<int> cartNeighbors;
20  myCartComm.CartNeighbors(cartNeighbors);
21  pcpp::report::CartesianSetup(messageStream,cartInfo);
22 
23  int myRank = myComm.Rank();
24  int nProc = myComm.Size();
25 
26  pcpp::io::Everyone(messageStream.str(),std::cout,testComm);
27  pcpp::io::RenewStream(messageStream);
28 
29  pcpp::IndexIntervalType globalExtent;
30  std::vector<size_t> globalSizes(3,101);
31  globalExtent.InitSimple(globalSizes);
32  pcpp::IndexIntervalType partitionExtent;
33 
34  messageStream << "Global size: ";
35  globalExtent.PrettyPrint(messageStream);
36  messageStream << std::endl;
37  if(pcpp::util::PartitionCartesianExtent(globalExtent,cartDims,cartCoords,partitionExtent,messageStream))
38  return;
39  messageStream << "Partition interval: ";
40  partitionExtent.PrettyPrint(messageStream);
41  messageStream << std::endl;
42 
43  pcpp::io::Everyone(messageStream.str(),std::cout,testComm);
44  pcpp::io::RenewStream(messageStream);
45 
46  size_t numX = partitionExtent[0].second - partitionExtent[0].first + 1;
47  size_t numY = partitionExtent[1].second - partitionExtent[1].first + 1;
48  size_t numZ = partitionExtent[2].second - partitionExtent[2].first + 1;
49  size_t nCells = (numX-1)*(numY-1)*(numZ-1);
50  size_t nNodes = numX*numY*numZ;
51  size_t numGlobalNodes = globalSizes[0]*globalSizes[1]*globalSizes[2];
52 
53  bool constructorworks = true;
54  simulation::state::halo testHalo(globalExtent,partitionExtent);
55  // testHalo.ConfigureHalo(globalExtent,partitionExtent);
56  if(testHalo.GlobalExtent() != globalExtent)
57  constructorworks = false;
58  if(testHalo.PartitionExtent() != partitionExtent)
59  constructorworks = false;
60 
61  std::vector<size_t> haloSizes(6,2);
62  std::vector<pcpp::IndexIntervalType> remoteHaloExtents(testHalo.CreateRemoteHaloExtents(haloSizes));
63  std::vector<pcpp::IndexIntervalType> localHaloExtents(testHalo.CreateLocalHaloExtents(haloSizes));
64 
65  bool createremotehaloextents = true;
66  bool createlocalhaloextents = true;
67 
68  if(remoteHaloExtents.size() != 6)
69  createremotehaloextents = false;
70  if(localHaloExtents.size() != 6)
71  createlocalhaloextents = false;
72 
73  if(!constructorworks){
74  myComm.SetErr(1);
75  }
76  if(myComm.Check())
77  constructorworks = false;
78 
79  if(!constructorworks)
80  return;
81 
82  int totalLocalHaloNodes = 0;
83  int totalRemoteHaloNodes = 0;
84 
85  std::vector<bool> haveLeftBoundary(3,false);
86  std::vector<bool> haveRightBoundary(3,false);
87 
88  for(int iDim = 0;iDim < 3;iDim++){
89  haveLeftBoundary[iDim] = (partitionExtent[iDim].first == 0);
90  haveRightBoundary[iDim] = (partitionExtent[iDim].second == globalExtent[iDim].second);
91  }
92 
93  for(int iDim = 0;iDim < 3;iDim++){
94 
95  pcpp::IndexIntervalType &leftRemoteHaloExtent(remoteHaloExtents[iDim*2]);
96  pcpp::IndexIntervalType &leftLocalHaloExtent(localHaloExtents[iDim*2]);
97 
98  if(haveLeftBoundary[iDim]){
99  if(!leftLocalHaloExtent.empty() ||
100  !leftRemoteHaloExtent.empty())
101  createremotehaloextents = false;
102  } else {
103 
104  totalLocalHaloNodes += leftLocalHaloExtent.NNodes();
105  totalRemoteHaloNodes += leftRemoteHaloExtent.NNodes();
106 
107  for(int hDim = 0;hDim < 3;hDim++){
108 
109  if(hDim == iDim){
110 
111  if(leftRemoteHaloExtent[iDim].first != (partitionExtent[iDim].first - haloSizes[iDim*2]))
112  createremotehaloextents = false;
113 
114  if(leftRemoteHaloExtent[iDim].second != (partitionExtent[iDim].first - 1))
115  createremotehaloextents = false;
116 
117  if(leftLocalHaloExtent[iDim].first != partitionExtent[iDim].first)
118  createlocalhaloextents = false;
119 
120  if(leftLocalHaloExtent[iDim].second != (partitionExtent[iDim].first + haloSizes[iDim*2] - 1))
121  createlocalhaloextents = false;
122 
123  } else {
124 
125  if(leftRemoteHaloExtent[hDim] != partitionExtent[hDim])
126  createremotehaloextents = false;
127 
128  if(leftLocalHaloExtent[hDim] != partitionExtent[hDim])
129  createlocalhaloextents = false;
130  }
131  }
132  }
133 
134  pcpp::IndexIntervalType &rightRemoteHaloExtent(remoteHaloExtents[iDim*2+1]);
135  pcpp::IndexIntervalType &rightLocalHaloExtent(localHaloExtents[iDim*2+1]);
136 
137  if(haveRightBoundary[iDim]){
138  if(!rightLocalHaloExtent.empty() ||
139  !rightRemoteHaloExtent.empty())
140  createremotehaloextents = false;
141  } else {
142 
143  totalLocalHaloNodes += rightLocalHaloExtent.NNodes();
144  totalRemoteHaloNodes += rightRemoteHaloExtent.NNodes();
145 
146 
147  for(int hDim = 0;hDim < 3;hDim++){
148 
149  if(hDim == iDim){
150 
151  if(rightRemoteHaloExtent[iDim].second != (partitionExtent[iDim].second + haloSizes[iDim*2]))
152  createremotehaloextents = false;
153 
154  if(rightRemoteHaloExtent[iDim].first != (partitionExtent[iDim].second + 1))
155  createremotehaloextents = false;
156 
157  if(rightLocalHaloExtent[iDim].second != partitionExtent[iDim].second)
158  createlocalhaloextents = false;
159 
160  if(rightLocalHaloExtent[iDim].first != (partitionExtent[iDim].second - haloSizes[iDim*2] + 1))
161  createlocalhaloextents = false;
162 
163  } else {
164 
165  if(rightRemoteHaloExtent[hDim] != partitionExtent[hDim])
166  createremotehaloextents = false;
167 
168  if(rightLocalHaloExtent[hDim] != partitionExtent[hDim])
169  createlocalhaloextents = false;
170  }
171  }
172  }
173  }
174 
175  if(!createremotehaloextents || !createlocalhaloextents)
176  myComm.SetErr(1);
177  if(myComm.Check()){
178  createremotehaloextents = false;
179  createlocalhaloextents = false;
180  }
181 
182  if(!createremotehaloextents || !createlocalhaloextents)
183  return;
184 
185  bool setremotehaloextents = false;
186  bool setlocalhaloextents = false;
187 
188  testHalo.SetRemoteHaloExtents(remoteHaloExtents);
189  if(testHalo.RemoteHaloExtents() == remoteHaloExtents)
190  setremotehaloextents = true;
191 
192  testHalo.SetLocalHaloExtents(localHaloExtents);
193  if(testHalo.LocalHaloExtents() == localHaloExtents)
194  setlocalhaloextents = true;
195 
196  if(!setremotehaloextents)
197  myComm.SetErr(1);
198  if(myComm.Check())
199  setremotehaloextents = false;
200 
201  myComm.SetErr(0);
202  if(!setlocalhaloextents)
203  myComm.SetErr(1);
204  if(myComm.Check())
205  setlocalhaloextents = false;
206 
207  if(!setlocalhaloextents || !setremotehaloextents)
208  return;
209 
210  simulation::state::base testState;
211  testState.AddField("time",'s',1,8,"s");
212  testState.AddField("U",'n',1,8,"stuff");
213  testState.AddField("V",'n',3,8,"things");
214  testState.AddField("iblank",'n',1,4,"");
215  testState.Create(nNodes,nCells);
216  int totalNumComponents = 4;
217 
218  std::vector<double> uVector(nNodes,0);
219  std::vector<double> vVector(nNodes*3,0);
220  testState.SetFieldBuffer("U",uVector);
221  testState.SetFieldBuffer("V",vVector);
222 
223 
224  for(size_t iNode = 0;iNode < nNodes;iNode++){
225 
226  size_t localK = iNode/(numX*numY);
227  size_t localJ = (iNode - (localK*numX*numY))/numX;
228  size_t localI = iNode%numX;
229 
230  size_t globalK = localK + partitionExtent[2].first;
231  size_t globalJ = localJ + partitionExtent[1].first;
232  size_t globalI = localI + partitionExtent[0].first;
233 
234  size_t globalNode = globalK * globalSizes[0]*globalSizes[1] +
235  globalJ * globalSizes[0] + globalI;
236 
237  uVector[iNode] = globalNode;
238  vVector[iNode] = uVector[iNode] + numGlobalNodes;
239  vVector[iNode+nNodes] = vVector[iNode] + numGlobalNodes;
240  vVector[iNode+(2*nNodes)] = vVector[iNode+nNodes] + numGlobalNodes;
241  }
242 
243 
244  std::vector<int> stateFieldIndices(2,1);
245  stateFieldIndices[1] = 2;
246  testState.SetStateFields(stateFieldIndices);
247 
248  bool configuredata = true;
249  testHalo.ConfigureData(testState);
250  if(testHalo.StateFieldIndices() != stateFieldIndices)
251  configuredata = false;
252  if(testHalo.numStateFields != 2)
253  configuredata = false;
254  if(testHalo.numStateComponents != 4)
255  configuredata = false;
256 
257  bool createSimpleSendBuffers = true;
258  bool createSimpleRecvBuffers = true;
259 
260  int numSend = testHalo.CreateSimpleSendBuffers();
261  int numRecv = testHalo.CreateSimpleRecvBuffers();
262 
263  std::vector<double *> &sendBuffers(testHalo.SendBuffers());
264  std::vector<double *> &recvBuffers(testHalo.RecvBuffers());
265 
266  if(sendBuffers.size() != localHaloExtents.size())
267  createSimpleSendBuffers = false;
268  if(recvBuffers.size() != remoteHaloExtents.size())
269  createSimpleRecvBuffers = false;
270  if(numSend != totalLocalHaloNodes*testHalo.numStateComponents)
271  createSimpleSendBuffers = false;
272  if(numRecv != totalRemoteHaloNodes*testHalo.numStateComponents)
273  createSimpleRecvBuffers = false;
274 
275  testHalo.CreateHaloBuffers();
276  bool createhalobuffers = true;
277 
278  // std::cout << "Posting simple recvs" << std::endl;
279  // pcpp::io::RenewStream(messageStream);
280  testHalo.PostSimpleReceives(cartNeighbors,myCartComm);
281 
282  // std::cout << "Packing simple sends" << std::endl;
283  bool packsimplesend = true;
284  testHalo.PackSimpleSendBuffers(testState);
285 
286  int numSendBuffers = sendBuffers.size();
287 
288  for(int iSend = 0;iSend < numSendBuffers;iSend++){
289 
290  pcpp::IndexIntervalType &sendExtent(localHaloExtents[iSend]);
291 
292  if(sendBuffers[iSend] != NULL){
293  std::vector<size_t> sendNodeIds;
294  partitionExtent.GetFlatIndices(sendExtent,sendNodeIds);
295 
296  size_t numSendNodes = sendExtent.NNodes();
297  size_t numValsSend = numSendNodes * testHalo.numStateComponents;
298 
299  for(size_t iVal = 0;iVal < numValsSend;iVal++){
300 
301  size_t componentId = iVal/numSendNodes;
302  size_t sendNodeId = iVal%numSendNodes;
303  size_t partitionNodeId = sendNodeIds[sendNodeId];
304 
305  size_t localK = partitionNodeId/(numX*numY);
306  size_t localJ = partitionNodeId/numX - localK*numY;
307  size_t localI = partitionNodeId%numX;
308 
309  size_t globalK = localK + partitionExtent[2].first;
310  size_t globalJ = localJ + partitionExtent[1].first;
311  size_t globalI = localI + partitionExtent[0].first;
312 
313  size_t globalNode = globalK * globalSizes[0]*globalSizes[1] +
314  globalJ * globalSizes[0] + globalI;
315  if(globalNode > numGlobalNodes)
316  exit(1);
317  double expectedValue = componentId*numGlobalNodes + globalNode;
318 
319  if(std::abs(sendBuffers[iSend][iVal] - expectedValue) > 1.0){
320  packsimplesend = false;
321  if(myRank != 0) {
322  std::cout << "Node ids (send/part/global): (" << sendNodeId << "," << partitionNodeId
323  << "," << globalNode
324  << ") Comp:" << componentId << " Value (exp/act): ("
325  << expectedValue << "," << sendBuffers[iSend][iVal] << ")" << std::endl;
326  }
327  }
328  }
329  } else {
330 
331  if(!sendExtent.empty()){
332  std::cout << "Send extent " << iSend << " had no associated send buffer."
333  << std::endl;
334  packsimplesend = false;
335  }
336 
337  }
338  }
339  if(!packsimplesend)
340  myComm.SetErr(1);
341  if(myComm.Check())
342  packsimplesend = false;
343 
344  if(!packsimplesend)
345  return;
346 
347  bool simplesend = true;
348  bool completesimplerecv = true;
349 
350  // std::cout << "Simple sends" << std::endl;
351  testHalo.SimpleSend(cartNeighbors,myCartComm);
352 
353  // std::cout << "Simple recv" << std::endl;
354  testHalo.CompleteSimpleReceives(myCartComm);
355 
356  // Check the halo buffers for the proper data to see if unpack succeeded
357  bool unpacksimplerecvbuffers = true;
358  std::vector<std::vector<double *> > &haloBuffers(testHalo.HaloBuffers());
359 
360 
361  for(int iDim = 0;iDim < 3;iDim++){
362  pcpp::IndexIntervalType &leftRemoteHaloExtent(remoteHaloExtents[iDim*2]);
363  if(!haveLeftBoundary[iDim]){
364  size_t numHaloNodes = leftRemoteHaloExtent.NNodes();
365  size_t kBegin = leftRemoteHaloExtent[2].first;
366  size_t kEnd = leftRemoteHaloExtent[2].second;
367  size_t jBegin = leftRemoteHaloExtent[1].first;
368  size_t jEnd = leftRemoteHaloExtent[1].second;
369  size_t iBegin = leftRemoteHaloExtent[0].first;
370  size_t iEnd = leftRemoteHaloExtent[0].second;
371  size_t haloNodeIndex = 0;
372  for(size_t kIndex = kBegin;kIndex <= kEnd;kIndex++){
373  for(size_t jIndex = jBegin;jIndex <= jEnd;jIndex++){
374  for(size_t iIndex = iBegin;iIndex <= iEnd;iIndex++){
375  size_t globalNode = kIndex * globalSizes[0]*globalSizes[1] +
376  jIndex * globalSizes[0] + iIndex;
377  if(globalNode > numGlobalNodes)
378  exit(1);
379  for(int iComponent = 0;iComponent < totalNumComponents;iComponent++){
380  int iField = (iComponent > 0 ? 1 : 0);
381  int compFac = (iField > 0 ? 1 : 0);
382  double expectedValue = iComponent*numGlobalNodes + globalNode;
383  size_t haloBufferIndex = compFac*(iComponent-1)*numHaloNodes+haloNodeIndex;
384  if(haloBuffers[iField][iDim*2][haloBufferIndex] != expectedValue){
385  messageStream << "R(" << myRank << ") N(" << iIndex << ","
386  << jIndex << "," << kIndex << ") V("
387  << haloBuffers[iField][iDim*2][haloBufferIndex]
388  << "," << expectedValue << ")" << std::endl;
389  unpacksimplerecvbuffers = false;
390  }
391  }
392  haloNodeIndex++;
393  }
394  }
395  }
396  }
397  pcpp::IndexIntervalType &rightRemoteHaloExtent(remoteHaloExtents[iDim*2+1]);
398  if(!haveRightBoundary[iDim]){
399  size_t numHaloNodes = rightRemoteHaloExtent.NNodes();
400  size_t kBegin = rightRemoteHaloExtent[2].first;
401  size_t kEnd = rightRemoteHaloExtent[2].second;
402  size_t jBegin = rightRemoteHaloExtent[1].first;
403  size_t jEnd = rightRemoteHaloExtent[1].second;
404  size_t iBegin = rightRemoteHaloExtent[0].first;
405  size_t iEnd = rightRemoteHaloExtent[0].second;
406  size_t haloNodeIndex = 0;
407  for(size_t kIndex = kBegin;kIndex <= kEnd;kIndex++){
408  for(size_t jIndex = jBegin;jIndex <= jEnd;jIndex++){
409  for(size_t iIndex = iBegin;iIndex <= iEnd;iIndex++){
410  size_t globalNode = kIndex * globalSizes[0]*globalSizes[1] +
411  jIndex * globalSizes[0] + iIndex;
412  if(globalNode > numGlobalNodes)
413  exit(1);
414  for(int iComponent = 0;iComponent < totalNumComponents;iComponent++){
415  int iField = (iComponent > 0 ? 1 : 0);
416  int compFac = (iField > 0 ? 1 : 0);
417  double expectedValue = iComponent*numGlobalNodes + globalNode;
418  size_t haloBufferIndex = compFac*(iComponent-1)*numHaloNodes+haloNodeIndex;
419  if(haloBuffers[iField][iDim*2+1][haloBufferIndex] != expectedValue){
420  messageStream << "R(" << myRank << ") N(" << iIndex << ","
421  << jIndex << "," << kIndex << ") V("
422  << haloBuffers[iField][iDim*2+1][haloBufferIndex]
423  << "," << expectedValue << ")" << std::endl;
424  unpacksimplerecvbuffers = false;
425  }
426  }
427  haloNodeIndex++;
428  }
429  }
430  }
431  }
432  }
433 
434  if(!unpacksimplerecvbuffers)
435  myComm.SetErr(1);
436  if(myComm.Check())
437  unpacksimplerecvbuffers = false;
438 
439  if(unpacksimplerecvbuffers){
440  parallelUnitResults.UpdateResult("Halo:SimpleSend",simplesend);
441  parallelUnitResults.UpdateResult("Halo:CompleteSimpleRecv",completesimplerecv);
442  }
443  parallelUnitResults.UpdateResult("Halo:UnpackSimpleRcvBuffers",unpacksimplerecvbuffers);
444 
445  // pcpp::io::Everyone(messageStream.str(),std::cout,myComm);
446 };
447 
448 void TestHaloPeriodic(ix::test::results &parallelUnitResults,pcpp::CommunicatorType &testComm)
449 {
450  pcpp::CommunicatorType myComm(testComm);
451  pcpp::CommunicatorType myCartComm;
452 
453  std::ostringstream messageStream;
454  pcpp::ParallelTopologyInfoType cartInfo;
455  cartInfo.cartDecompDirections.resize(3,1);
456  // cartInfo.cartDecompDirections[2] = 1;
457  cartInfo.isPeriodic.resize(3,1);
458  // cartInfo.isPeriodic[2] = 1;
459  pcpp::comm::SetupCartesianTopology(myComm,cartInfo,myCartComm,messageStream);
460  std::vector<int> &cartCoords(myCartComm.CartCoordinates());
461  std::vector<int> &cartDims(myCartComm.CartDimensions());
462  std::vector<int> cartNeighbors;
463  myCartComm.CartNeighbors(cartNeighbors);
464  pcpp::report::CartesianSetup(messageStream,cartInfo);
465 
466  int myRank = myComm.Rank();
467  int nProc = myComm.Size();
468 
469  if(myRank == 0){
470  std::cout << messageStream.str() << std::endl;
471  pcpp::io::RenewStream(messageStream);
472  }
473 
474  pcpp::IndexIntervalType globalExtent;
475  std::vector<size_t> globalSizes(3,101);
476  // globalSizes[0] = 1;
477  // globalSizes[1] = 1;
478 
479  globalExtent.InitSimple(globalSizes);
480  pcpp::IndexIntervalType partitionExtent;
481 
482  messageStream << "Global size: ";
483  globalExtent.PrettyPrint(messageStream);
484  messageStream << std::endl;
485  if(pcpp::util::PartitionCartesianExtent(globalExtent,cartDims,cartCoords,partitionExtent,messageStream))
486  return;
487  messageStream << "Partition interval: ";
488  partitionExtent.PrettyPrint(messageStream);
489  messageStream << std::endl;
490  if(myRank == 0){
491  std::cout << messageStream.str() << std::endl;
492  pcpp::io::RenewStream(messageStream);
493  }
494 
495  size_t numX = partitionExtent[0].second - partitionExtent[0].first + 1;
496  size_t numY = partitionExtent[1].second - partitionExtent[1].first + 1;
497  size_t numZ = partitionExtent[2].second - partitionExtent[2].first + 1;
498  size_t nCells = (numX-1)*(numY-1)*(numZ-1);
499  size_t nNodes = numX*numY*numZ;
500  size_t numGlobalNodes = globalSizes[0]*globalSizes[1]*globalSizes[2];
501 
502  simulation::state::halo testHalo(globalExtent,partitionExtent);
503  // testHalo.ConfigureHalo(globalExtent,partitionExtent);
504  std::vector<bool> haveNeighbors(6,false);
505  for(int iN = 0;iN < 6;iN++)
506  haveNeighbors[iN] = (cartNeighbors[iN] >= 0);
507  testHalo.SetNeighbors(haveNeighbors);
508 
509  std::vector<size_t> haloSizes(6,2);
510  std::vector<pcpp::IndexIntervalType> remoteHaloExtents(testHalo.CreateRemoteHaloExtents(haloSizes));
511  std::vector<pcpp::IndexIntervalType> localHaloExtents(testHalo.CreateLocalHaloExtents(haloSizes));
512 
513  int totalLocalHaloNodes = 0;
514  int totalRemoteHaloNodes = 0;
515 
516  std::vector<bool> haveLeftBoundary(3,false);
517  std::vector<bool> haveRightBoundary(3,false);
518 
519  for(int iDim = 0;iDim < 3;iDim++){
520  haveLeftBoundary[iDim] = ((partitionExtent[iDim].first == 0) && !haveNeighbors[2*iDim]);
521  haveRightBoundary[iDim] = ((partitionExtent[iDim].second == globalExtent[iDim].second) && !haveNeighbors[2*iDim+1]);
522  }
523 
524  bool leftremotehaloextent = true;
525  bool leftlocalhaloextent = true;
526  bool periodicleftremotehaloextent = true;
527  bool periodicleftlocalhaloextent = true;
528  bool createleftremotehaloextents = true;
529  bool createleftlocalhaloextents = true;
530 
531  bool rightremotehaloextent = true;
532  bool rightlocalhaloextent = true;
533  bool periodicrightremotehaloextent = true;
534  bool periodicrightlocalhaloextent = true;
535  bool createrightremotehaloextents = true;
536  bool createrightlocalhaloextents = true;
537 
538 
539  for(int iDim = 0;iDim < 3;iDim++){
540 
541  pcpp::IndexIntervalType &leftRemoteHaloExtent(remoteHaloExtents[iDim*2]);
542  pcpp::IndexIntervalType &leftLocalHaloExtent(localHaloExtents[iDim*2]);
543 
544  messageStream << "LeftRemoteHalo(" << iDim << "): ";
545  leftRemoteHaloExtent.PrettyPrint(messageStream);
546  messageStream << std::endl;
547  messageStream << "LeftLocalHalo(" << iDim << "): ";
548  leftLocalHaloExtent.PrettyPrint(messageStream);
549  messageStream << std::endl;
550 
551  if(haveLeftBoundary[iDim]){
552  if(!leftLocalHaloExtent.empty())
553  createleftlocalhaloextents = false;
554  if(!leftRemoteHaloExtent.empty())
555  createleftremotehaloextents = false;
556  } else {
557 
558  totalLocalHaloNodes += leftLocalHaloExtent.NNodes();
559  totalRemoteHaloNodes += leftRemoteHaloExtent.NNodes();
560 
561  for(int hDim = 0;hDim < 3;hDim++){
562 
563  if(hDim == iDim){
564 
565  if(partitionExtent[iDim].first == 0){
566 
567  if(leftRemoteHaloExtent[iDim].first != (globalExtent[iDim].second - haloSizes[iDim*2] + 1))
568  periodicleftremotehaloextent = false;
569  if(leftRemoteHaloExtent[iDim].second != (globalExtent[iDim].second))
570  periodicleftremotehaloextent = false;
571  if(leftLocalHaloExtent[iDim].first != partitionExtent[iDim].first)
572  periodicleftlocalhaloextent = false;
573  if(leftLocalHaloExtent[iDim].second != (partitionExtent[iDim].first + haloSizes[iDim*2] - 1))
574  periodicleftlocalhaloextent = false;
575  } else {
576  if(leftRemoteHaloExtent[iDim].first != (partitionExtent[iDim].first - haloSizes[iDim*2]))
577  leftremotehaloextent = false;
578  if(leftRemoteHaloExtent[iDim].second != (partitionExtent[iDim].first - 1))
579  leftremotehaloextent = false;
580  if(leftLocalHaloExtent[iDim].first != partitionExtent[iDim].first)
581  leftlocalhaloextent = false;
582  if(leftLocalHaloExtent[iDim].second != (partitionExtent[iDim].first + haloSizes[iDim*2] - 1))
583  leftlocalhaloextent = false;
584  }
585 
586  } else {
587 
588  if(partitionExtent[iDim].first == 0){
589 
590  if(leftRemoteHaloExtent[hDim] != partitionExtent[hDim])
591  periodicleftremotehaloextent = false;
592  if(leftLocalHaloExtent[hDim] != partitionExtent[hDim])
593  periodicleftlocalhaloextent = false;
594 
595  } else {
596 
597  if(leftRemoteHaloExtent[hDim] != partitionExtent[hDim])
598  leftremotehaloextent = false;
599  if(leftLocalHaloExtent[hDim] != partitionExtent[hDim])
600  leftlocalhaloextent = false;
601  }
602  }
603  }
604  }
605 
606  pcpp::IndexIntervalType &rightRemoteHaloExtent(remoteHaloExtents[iDim*2+1]);
607  pcpp::IndexIntervalType &rightLocalHaloExtent(localHaloExtents[iDim*2+1]);
608 
609  messageStream << "RightRemoteHalo(" << iDim << "): ";
610  rightRemoteHaloExtent.PrettyPrint(messageStream);
611  messageStream << std::endl;
612  messageStream << "RightLocalHalo(" << iDim << "): ";
613  rightLocalHaloExtent.PrettyPrint(messageStream);
614  messageStream << std::endl;
615 
616  if(haveRightBoundary[iDim]){
617  if(!rightLocalHaloExtent.empty())
618  createrightlocalhaloextents = false;
619  if(!rightRemoteHaloExtent.empty())
620  createrightremotehaloextents = false;
621  } else {
622 
623 
624  totalLocalHaloNodes += rightLocalHaloExtent.NNodes();
625  totalRemoteHaloNodes += rightRemoteHaloExtent.NNodes();
626 
627 
628  for(int hDim = 0;hDim < 3;hDim++){
629 
630  if(hDim == iDim){
631 
632  if(partitionExtent[iDim].second == globalExtent[iDim].second){
633  if(rightRemoteHaloExtent[iDim].second != (haloSizes[iDim*2+1]-1))
634  periodicrightremotehaloextent = false;
635  if(rightRemoteHaloExtent[iDim].first != 0)
636  periodicrightremotehaloextent = false;
637  if(rightLocalHaloExtent[iDim].second != partitionExtent[iDim].second)
638  periodicrightlocalhaloextent = false;
639  if(rightLocalHaloExtent[iDim].first != (partitionExtent[iDim].second - haloSizes[iDim*2+1] + 1))
640  periodicrightlocalhaloextent = false;
641 
642  } else {
643 
644  if(rightRemoteHaloExtent[iDim].second != (partitionExtent[iDim].second + haloSizes[iDim*2+1]))
645  rightremotehaloextent = false;
646  if(rightRemoteHaloExtent[iDim].first != (partitionExtent[iDim].second + 1))
647  rightremotehaloextent = false;
648  if(rightLocalHaloExtent[iDim].second != partitionExtent[iDim].second)
649  rightlocalhaloextent = false;
650  if(rightLocalHaloExtent[iDim].first != (partitionExtent[iDim].second - haloSizes[iDim*2+1] + 1))
651  rightlocalhaloextent = false;
652  }
653 
654  } else {
655 
656  if(partitionExtent[iDim].second == globalExtent[iDim].second){
657 
658  if(rightRemoteHaloExtent[hDim] != partitionExtent[hDim])
659  periodicrightremotehaloextent = false;
660  if(rightLocalHaloExtent[hDim] != partitionExtent[hDim])
661  periodicrightlocalhaloextent = false;
662 
663  } else {
664 
665  if(rightRemoteHaloExtent[hDim] != partitionExtent[hDim])
666  rightremotehaloextent = false;
667  if(rightLocalHaloExtent[hDim] != partitionExtent[hDim])
668  rightlocalhaloextent = false;
669  }
670  }
671  }
672  }
673  }
674 
675  leftremotehaloextent = CheckResult(leftremotehaloextent,testComm);
676  leftlocalhaloextent = CheckResult(leftlocalhaloextent,testComm);
677  periodicleftremotehaloextent = CheckResult(periodicleftremotehaloextent,testComm);
678  periodicleftlocalhaloextent = CheckResult(periodicleftlocalhaloextent,testComm);
679  createleftremotehaloextents = CheckResult(createleftremotehaloextents,testComm);
680  createleftlocalhaloextents = CheckResult(createleftlocalhaloextents,testComm);
681 
682  parallelUnitResults.UpdateResult("Halo:LRemoteHalo",leftremotehaloextent);
683  parallelUnitResults.UpdateResult("Halo:LLocalHalo",leftlocalhaloextent);
684  parallelUnitResults.UpdateResult("Halo:PeriodicLRemoteHalo",periodicleftremotehaloextent);
685  parallelUnitResults.UpdateResult("Halo:PeriodicLLocalHalo",periodicleftlocalhaloextent);
686  parallelUnitResults.UpdateResult("Halo:CreateLRemoteHalo",createleftremotehaloextents);
687  parallelUnitResults.UpdateResult("Halo:CreateLLocalHalo",createleftlocalhaloextents);
688 
689  rightremotehaloextent = CheckResult(rightremotehaloextent,testComm);
690  rightlocalhaloextent = CheckResult(rightlocalhaloextent,testComm);
691  periodicrightremotehaloextent = CheckResult(periodicrightremotehaloextent,testComm);
692  periodicrightlocalhaloextent = CheckResult(periodicrightlocalhaloextent,testComm);
693  createrightremotehaloextents = CheckResult(createrightremotehaloextents,testComm);
694  createrightlocalhaloextents = CheckResult(createrightlocalhaloextents,testComm);
695 
696  parallelUnitResults.UpdateResult("Halo:RRemoteHalo",rightremotehaloextent);
697  parallelUnitResults.UpdateResult("Halo:RLocalHalo",rightlocalhaloextent);
698  parallelUnitResults.UpdateResult("Halo:PeriodicRRemoteHalo",periodicrightremotehaloextent);
699  parallelUnitResults.UpdateResult("Halo:PeriodicRLocalHalo",periodicrightlocalhaloextent);
700  parallelUnitResults.UpdateResult("Halo:CreateRRemoteHalo",createrightremotehaloextents);
701  parallelUnitResults.UpdateResult("Halo:CreateRLocalHalo",createrightlocalhaloextents);
702 
703  pcpp::io::Everyone(messageStream.str(),std::cout,myComm);
704  pcpp::io::RenewStream(messageStream);
705 
706  testHalo.SetRemoteHaloExtents(remoteHaloExtents);
707  testHalo.SetLocalHaloExtents(localHaloExtents);
708 
709 
710  simulation::state::base testState;
711  testState.AddField("time",'s',1,8,"s");
712  testState.AddField("U",'n',1,8,"stuff");
713  testState.AddField("V",'n',3,8,"things");
714  testState.AddField("iblank",'n',1,4,"");
715  testState.Create(nNodes,nCells);
716  int totalNumComponents = 4;
717 
718  std::vector<double> uVector(nNodes,0);
719  std::vector<double> vVector(nNodes*3,0);
720  testState.SetFieldBuffer("U",uVector);
721  testState.SetFieldBuffer("V",vVector);
722 
723 
724  for(size_t iNode = 0;iNode < nNodes;iNode++){
725 
726  size_t localK = iNode/(numX*numY);
727  size_t localJ = (iNode - (localK*numX*numY))/numX;
728  size_t localI = iNode%numX;
729 
730  size_t globalK = localK + partitionExtent[2].first;
731  size_t globalJ = localJ + partitionExtent[1].first;
732  size_t globalI = localI + partitionExtent[0].first;
733 
734  size_t globalNode = globalK * globalSizes[0]*globalSizes[1] +
735  globalJ * globalSizes[0] + globalI;
736 
737  uVector[iNode] = globalNode;
738  vVector[iNode] = uVector[iNode] + numGlobalNodes;
739  vVector[iNode+nNodes] = vVector[iNode] + numGlobalNodes;
740  vVector[iNode+(2*nNodes)] = vVector[iNode+nNodes] + numGlobalNodes;
741  }
742 
743 
744  std::vector<int> stateFieldIndices(2,1);
745  stateFieldIndices[1] = 2;
746  testState.SetStateFields(stateFieldIndices);
747  testHalo.ConfigureData(testState);
748 
749  int numSend = testHalo.CreateSimpleSendBuffers();
750  int numRecv = testHalo.CreateSimpleRecvBuffers();
751 
752  if(testHalo.numStateComponents == 0){
753  std::cout << "ERROR WITH HALO STATE COMPONENT COUNT!!" << std::endl;
754  return;
755  }
756  if(totalRemoteHaloNodes == 0 || totalLocalHaloNodes == 0){
757  std::cout << "ERROR WITH NO HALO NODES!!" << std::endl;
758  return;
759  }
760  if(numSend == 0 || numRecv == 0){
761  std::cout << "ERROR WITH EMPTY COMM BUFFERS!!" << std::endl;
762  return;
763  }
764 
765  bool createSimpleSendBuffers = true;
766  bool createSimpleRecvBuffers = true;
767 
768  std::vector<double *> &sendBuffers(testHalo.SendBuffers());
769  std::vector<double *> &recvBuffers(testHalo.RecvBuffers());
770 
771  if(sendBuffers.size() != localHaloExtents.size())
772  createSimpleSendBuffers = false;
773  if(recvBuffers.size() != remoteHaloExtents.size())
774  createSimpleRecvBuffers = false;
775  if(numSend != totalLocalHaloNodes*testHalo.numStateComponents)
776  createSimpleSendBuffers = false;
777  if(numRecv != totalRemoteHaloNodes*testHalo.numStateComponents)
778  createSimpleRecvBuffers = false;
779 
780  testHalo.CreateHaloBuffers();
781 
782  CheckResult(createSimpleSendBuffers,testComm);
783  CheckResult(createSimpleRecvBuffers,testComm);
784  if(!createSimpleSendBuffers || !createSimpleRecvBuffers){
785  std::cout << "ERROR CREATING PERIODIC COMM BUFFERS!" << std::endl;
786  return;
787  }
788 
789  testHalo.PostSimpleReceives(cartNeighbors,myCartComm);
790  bool packsimplesend = true;
791  testHalo.PackSimpleSendBuffers(testState);
792 
793  int numSendBuffers = sendBuffers.size();
794 
795  for(int iSend = 0;iSend < numSendBuffers;iSend++){
796 
797  pcpp::IndexIntervalType &sendExtent(localHaloExtents[iSend]);
798 
799  if(sendBuffers[iSend] != NULL){
800 
801  messageStream << "Send Address(" << iSend << "): "
802  << sendBuffers[iSend] << std::endl;
803 
804  if(sendExtent.empty()){
805  std::cout << "Send extent " << iSend << " is empty, but "
806  << "has associated send buffer!!" << std::endl;
807  createSimpleSendBuffers = false;
808  }
809 
810  std::vector<size_t> sendNodeIds;
811  partitionExtent.GetFlatIndices(sendExtent,sendNodeIds);
812 
813  size_t numSendNodes = sendExtent.NNodes();
814  size_t numValsSend = numSendNodes * testHalo.numStateComponents;
815 
816  for(size_t iVal = 0;iVal < numValsSend;iVal++){
817 
818  size_t componentId = iVal/numSendNodes;
819  size_t sendNodeId = iVal%numSendNodes;
820  size_t partitionNodeId = sendNodeIds[sendNodeId];
821 
822  size_t localK = partitionNodeId/(numX*numY);
823  size_t localJ = partitionNodeId/numX - localK*numY;
824  size_t localI = partitionNodeId%numX;
825 
826  size_t globalK = localK + partitionExtent[2].first;
827  size_t globalJ = localJ + partitionExtent[1].first;
828  size_t globalI = localI + partitionExtent[0].first;
829 
830  size_t globalNode = globalK * globalSizes[0]*globalSizes[1] +
831  globalJ * globalSizes[0] + globalI;
832  if(globalNode > numGlobalNodes){
833  std::cout << "ERROR COUNTING NODES!!" << std::endl;
834  exit(1);
835  }
836 
837  double expectedValue = componentId*numGlobalNodes + globalNode;
838 
839  if(std::abs(sendBuffers[iSend][iVal] - expectedValue) > .5){
840  packsimplesend = false;
841  if(myRank != 0) {
842  std::cout << "Node ids (send/part/global): (" << sendNodeId << "," << partitionNodeId
843  << "," << globalNode
844  << ") Comp:" << componentId << " Value (exp/act): ("
845  << expectedValue << "," << sendBuffers[iSend][iVal] << ")" << std::endl;
846  }
847  }
848  }
849  } else {
850 
851  if(!sendExtent.empty()){
852  std::cout << "Send extent " << iSend << " had no associated send buffer."
853  << std::endl;
854  createSimpleSendBuffers = false;
855  }
856  }
857  }
858 
859  CheckResult(createSimpleSendBuffers,testComm);
860  parallelUnitResults.UpdateResult("Halo:PeriodicSendBuffers",createSimpleSendBuffers);
861 
862  CheckResult(packsimplesend,testComm);
863  parallelUnitResults.UpdateResult("Halo:PackPeriodicSends",packsimplesend);
864 
865  if(!packsimplesend || !createSimpleSendBuffers)
866  return;
867 
868  bool simplesend = true;
869  bool completesimplerecv = true;
870 
871  testHalo.SimpleSend(cartNeighbors,myCartComm);
872  testHalo.CompleteSimpleReceives(myCartComm);
873 
874  // Check the recv buffers for the proper data to see if the send worked
875  int numRecvBuffers = recvBuffers.size();
876  bool periodiccomm = true;
877  for(int iRecv = 0;iRecv < numSendBuffers;iRecv++){
878 
879  pcpp::IndexIntervalType &recvExtent(remoteHaloExtents[iRecv]);
880  if(recvExtent.empty())
881  continue;
882  // iRecv corresponds to the [-,+] directions in each dimension x[0,1] y[2,3] z[4,5]
883  double *recvBuffer = recvBuffers[iRecv];
884  messageStream << "Recv Address(" << iRecv << "): "
885  << recvBuffer << std::endl;
886  size_t numRecvNodes = recvExtent.NNodes();
887  size_t baseRecvIndex = 0;
888  size_t kStart = remoteHaloExtents[iRecv][2].first;
889  size_t kEnd = remoteHaloExtents[iRecv][2].second;
890  size_t jStart = remoteHaloExtents[iRecv][1].first;
891  size_t jEnd = remoteHaloExtents[iRecv][1].second;
892  size_t iStart = remoteHaloExtents[iRecv][0].first;
893  size_t iEnd = remoteHaloExtents[iRecv][0].second;
894  for(size_t haloK = kStart;haloK <= kEnd;haloK++){
895  for(size_t haloJ = jStart;haloJ <= jEnd;haloJ++){
896  for(size_t haloI = iStart;haloI <= iEnd;haloI++){
897  size_t globalNode = haloK*globalSizes[0]*globalSizes[1] +
898  haloJ*globalSizes[0] + haloI;
899  for(int iComponent = 0;iComponent < totalNumComponents;iComponent++){
900  double expectedValue = globalNode + iComponent*numGlobalNodes;
901  size_t recvBufferIndex = baseRecvIndex + iComponent*numRecvNodes;
902  double actualValue = recvBuffer[recvBufferIndex];
903  if(actualValue != expectedValue){
904  messageStream << "Rank(" << myRank << ") Dir(" << iRecv <<") GlobalNode("
905  << globalNode << ") Node[comp"
906  << iComponent << ",n" << baseRecvIndex << ",i" << recvBufferIndex
907  << "]g(" << haloI << "," << haloJ << "," << haloK << ") RecvValue(actual,expected):("
908  << actualValue << "," << expectedValue << ")" << std::endl;
909  periodiccomm = false;
910  }
911  }
912  baseRecvIndex++;
913  }
914  }
915  }
916  }
917  CheckResult(periodiccomm,testComm);
918  parallelUnitResults.UpdateResult("Halo:PeriodicCommunication",periodiccomm);
919  pcpp::io::Everyone(messageStream.str(),std::cout,testComm);
920  pcpp::io::RenewStream(messageStream);
921  if(!periodiccomm)
922  return;
923 
924  // Check the halo buffers for the proper data to see if unpack succeeded
925  bool unpacksimplerecvbuffers = true;
926  std::vector<std::vector<double *> > &haloBuffers(testHalo.HaloBuffers());
927 
928 
929  for(int iDim = 0;iDim < 3;iDim++){
930  pcpp::IndexIntervalType &leftRemoteHaloExtent(remoteHaloExtents[iDim*2]);
931  if(!haveLeftBoundary[iDim]){
932  size_t numHaloNodes = leftRemoteHaloExtent.NNodes();
933  size_t kBegin = leftRemoteHaloExtent[2].first;
934  size_t kEnd = leftRemoteHaloExtent[2].second;
935  size_t jBegin = leftRemoteHaloExtent[1].first;
936  size_t jEnd = leftRemoteHaloExtent[1].second;
937  size_t iBegin = leftRemoteHaloExtent[0].first;
938  size_t iEnd = leftRemoteHaloExtent[0].second;
939  size_t haloNodeIndex = 0;
940  for(size_t kIndex = kBegin;kIndex <= kEnd;kIndex++){
941  for(size_t jIndex = jBegin;jIndex <= jEnd;jIndex++){
942  for(size_t iIndex = iBegin;iIndex <= iEnd;iIndex++){
943  size_t globalNode = kIndex * globalSizes[0]*globalSizes[1] +
944  jIndex * globalSizes[0] + iIndex;
945  if(globalNode > numGlobalNodes)
946  exit(1);
947  for(int iComponent = 0;iComponent < totalNumComponents;iComponent++){
948  int iField = (iComponent > 0 ? 1 : 0);
949  int compFac = (iField > 0 ? 1 : 0);
950  double expectedValue = iComponent*numGlobalNodes + globalNode;
951  size_t haloBufferIndex = compFac*(iComponent-1)*numHaloNodes+haloNodeIndex;
952  if(haloBuffers[iField][iDim*2][haloBufferIndex] != expectedValue){
953  messageStream << "Rank(" << myRank << ") GlobalNode(" << globalNode << ") LHaloNode["
954  << iField << "," << haloBufferIndex << "](" << iIndex << ","
955  << jIndex << "," << kIndex << ") HaloValue(actual,expected):("
956  << haloBuffers[iField][iDim*2][haloBufferIndex]
957  << "," << expectedValue << ")" << std::endl;
958  unpacksimplerecvbuffers = false;
959  }
960  }
961  haloNodeIndex++;
962  }
963  }
964  }
965  }
966  pcpp::IndexIntervalType &rightRemoteHaloExtent(remoteHaloExtents[iDim*2+1]);
967  if(!haveRightBoundary[iDim]){
968  size_t numHaloNodes = rightRemoteHaloExtent.NNodes();
969  size_t kBegin = rightRemoteHaloExtent[2].first;
970  size_t kEnd = rightRemoteHaloExtent[2].second;
971  size_t jBegin = rightRemoteHaloExtent[1].first;
972  size_t jEnd = rightRemoteHaloExtent[1].second;
973  size_t iBegin = rightRemoteHaloExtent[0].first;
974  size_t iEnd = rightRemoteHaloExtent[0].second;
975  size_t haloNodeIndex = 0;
976  for(size_t kIndex = kBegin;kIndex <= kEnd;kIndex++){
977  for(size_t jIndex = jBegin;jIndex <= jEnd;jIndex++){
978  for(size_t iIndex = iBegin;iIndex <= iEnd;iIndex++){
979  size_t globalNode = kIndex * globalSizes[0]*globalSizes[1] +
980  jIndex * globalSizes[0] + iIndex;
981  if(globalNode > numGlobalNodes)
982  exit(1);
983  for(int iComponent = 0;iComponent < totalNumComponents;iComponent++){
984  int iField = (iComponent > 0 ? 1 : 0);
985  int compFac = (iField > 0 ? 1 : 0);
986  double expectedValue = iComponent*numGlobalNodes + globalNode;
987  size_t haloBufferIndex = compFac*(iComponent-1)*numHaloNodes+haloNodeIndex;
988  if(haloBuffers[iField][iDim*2+1][haloBufferIndex] != expectedValue){
989  messageStream << "Rank(" << myRank << ") GlobalNode(" << globalNode << ") RHaloNode["
990  << iField << "," << haloBufferIndex << "](" << iIndex << ","
991  << jIndex << "," << kIndex << ") HaloValue(actual,expected):("
992  << haloBuffers[iField][iDim*2+1][haloBufferIndex]
993  << "," << expectedValue << ")" << std::endl;
994  unpacksimplerecvbuffers = false;
995  }
996  }
997  haloNodeIndex++;
998  }
999  }
1000  }
1001  }
1002  }
1003 
1004  CheckResult(unpacksimplerecvbuffers,testComm);
1005  CheckResult(simplesend,testComm);
1006  CheckResult(completesimplerecv,testComm);
1007 
1008  parallelUnitResults.UpdateResult("Halo:PeriodicSend",simplesend);
1009  parallelUnitResults.UpdateResult("Halo:PeriodicCompleteSimpleRecv",completesimplerecv);
1010  parallelUnitResults.UpdateResult("Halo:PeriodicUnpackSimpleRecv",unpacksimplerecvbuffers);
1011 
1012  if(!unpacksimplerecvbuffers)
1013  pcpp::io::Everyone(messageStream.str(),std::cout,testComm);
1014 };
1015 
1016 void TestHaloBasic(ix::test::results &serialUnitResults) {
1017 
1018  pcpp::IndexIntervalType globalExtent;
1019  std::vector<size_t> globalSizes(3,101);
1020  globalExtent.InitSimple(globalSizes);
1021 
1022  pcpp::IndexIntervalType partitionExtent;
1023  std::vector<size_t> startIndices(3,5);
1024  std::vector<size_t> partitionSizes(3,6);
1025  partitionExtent.Init(startIndices,partitionSizes);
1026 
1027  size_t numX = 6;
1028  size_t numY = 6;
1029  size_t numZ = 6;
1030  size_t nCells = 125;
1031  size_t nNodes = 216;
1032 
1033  bool constructorworks = true;
1034  simulation::state::halo testHalo(globalExtent,partitionExtent);
1035  // testHalo.ConfigureHalo(globalExtent,partitionExtent);
1036  if(testHalo.GlobalExtent() != globalExtent)
1037  constructorworks = false;
1038  if(testHalo.PartitionExtent() != partitionExtent)
1039  constructorworks = false;
1040 
1041 
1042  std::vector<size_t> haloSizes(6,2);
1043  std::vector<pcpp::IndexIntervalType> remoteHaloExtents(testHalo.CreateRemoteHaloExtents(haloSizes));
1044  std::vector<pcpp::IndexIntervalType> localHaloExtents(testHalo.CreateLocalHaloExtents(haloSizes));
1045 
1046  bool createremotehaloextents = true;
1047  bool createlocalhaloextents = true;
1048 
1049  if(remoteHaloExtents.size() != 6)
1050  createremotehaloextents = false;
1051  if(localHaloExtents.size() != 6)
1052  createlocalhaloextents = false;
1053 
1054 
1055  int totalLocalHaloNodes = 0;
1056  int totalRemoteHaloNodes = 0;
1057 
1058  for(int iDim = 0;iDim < 3;iDim++){
1059 
1060  pcpp::IndexIntervalType &leftRemoteHaloExtent(remoteHaloExtents[iDim*2]);
1061  pcpp::IndexIntervalType &rightRemoteHaloExtent(remoteHaloExtents[iDim*2+1]);
1062  pcpp::IndexIntervalType &leftLocalHaloExtent(localHaloExtents[iDim*2]);
1063  pcpp::IndexIntervalType &rightLocalHaloExtent(localHaloExtents[iDim*2+1]);
1064 
1065  totalLocalHaloNodes += leftLocalHaloExtent.NNodes();
1066  totalLocalHaloNodes += rightLocalHaloExtent.NNodes();
1067  totalRemoteHaloNodes += leftRemoteHaloExtent.NNodes();
1068  totalRemoteHaloNodes += rightRemoteHaloExtent.NNodes();
1069 
1070  for(int hDim = 0;hDim < 3;hDim++){
1071 
1072  if(hDim == iDim){
1073 
1074  if(leftRemoteHaloExtent[iDim].first != (partitionExtent[iDim].first - haloSizes[iDim*2]))
1075  createremotehaloextents = false;
1076  if(leftRemoteHaloExtent[iDim].second != (partitionExtent[iDim].first - 1))
1077  createremotehaloextents = false;
1078 
1079  if(leftLocalHaloExtent[iDim].first != partitionExtent[iDim].first)
1080  createlocalhaloextents = false;
1081  if(leftLocalHaloExtent[iDim].second != (partitionExtent[iDim].first + haloSizes[iDim*2] - 1))
1082  createlocalhaloextents = false;
1083 
1084  if(rightRemoteHaloExtent[iDim].second != (partitionExtent[iDim].second + haloSizes[iDim*2]))
1085  createremotehaloextents = false;
1086  if(rightRemoteHaloExtent[iDim].first != (partitionExtent[iDim].second + 1))
1087  createremotehaloextents = false;
1088 
1089  if(rightLocalHaloExtent[iDim].second != partitionExtent[iDim].second)
1090  createlocalhaloextents = false;
1091  if(rightLocalHaloExtent[iDim].first != (partitionExtent[iDim].second - haloSizes[iDim*2] + 1))
1092  createlocalhaloextents = false;
1093 
1094  } else {
1095 
1096  if(leftRemoteHaloExtent[hDim] != partitionExtent[hDim])
1097  createremotehaloextents = false;
1098  if(rightRemoteHaloExtent[hDim] != partitionExtent[hDim])
1099  createremotehaloextents = false;
1100 
1101  if(leftLocalHaloExtent[hDim] != partitionExtent[hDim])
1102  createlocalhaloextents = false;
1103  if(rightLocalHaloExtent[hDim] != partitionExtent[hDim])
1104  createlocalhaloextents = false;
1105  }
1106  }
1107  }
1108 
1109  bool setremotehaloextents = false;
1110  bool setlocalhaloextents = false;
1111 
1112  testHalo.SetRemoteHaloExtents(remoteHaloExtents);
1113  if(testHalo.RemoteHaloExtents() == remoteHaloExtents)
1114  setremotehaloextents = true;
1115 
1116  testHalo.SetLocalHaloExtents(localHaloExtents);
1117  if(testHalo.LocalHaloExtents() == localHaloExtents)
1118  setlocalhaloextents = true;
1119 
1120  simulation::state::base testState;
1121  testState.AddField("time",'s',1,8,"s");
1122  testState.AddField("U",'n',1,8,"stuff");
1123  testState.AddField("V",'n',3,8,"things");
1124  testState.AddField("iblank",'n',1,4,"");
1125  testState.Create(nNodes,nCells);
1126 
1127  std::vector<double> uVector(nNodes,0);
1128  std::vector<double> vVector(nNodes*3,0);
1129  testState.SetFieldBuffer("U",uVector);
1130  testState.SetFieldBuffer("V",vVector);
1131 
1132  for(int iNode = 0;iNode < nNodes;iNode++){
1133  uVector[iNode] = iNode;
1134  vVector[iNode] = uVector[iNode] + nNodes;
1135  vVector[iNode+nNodes] = vVector[iNode] + nNodes;
1136  vVector[iNode+(2*nNodes)] = vVector[iNode+nNodes] + nNodes;
1137  }
1138 
1139  std::vector<int> stateFieldIndices(2,1);
1140  stateFieldIndices[1] = 2;
1141  testState.SetStateFields(stateFieldIndices);
1142 
1143  bool configuredata = true;
1144  testHalo.ConfigureData(testState);
1145  if(testHalo.StateFieldIndices() != stateFieldIndices)
1146  configuredata = false;
1147  if(testHalo.numStateFields != 2)
1148  configuredata = false;
1149  if(testHalo.numStateComponents != 4)
1150  configuredata = false;
1151 
1152  bool createSimpleSendBuffers = true;
1153  bool createSimpleRecvBuffers = true;
1154 
1155  int numSend = testHalo.CreateSimpleSendBuffers();
1156  int numRecv = testHalo.CreateSimpleRecvBuffers();
1157 
1158  std::vector<double *> &sendBuffers(testHalo.SendBuffers());
1159  std::vector<double *> &recvBuffers(testHalo.RecvBuffers());
1160 
1161  if(sendBuffers.size() != localHaloExtents.size())
1162  createSimpleSendBuffers = false;
1163  if(recvBuffers.size() != remoteHaloExtents.size())
1164  createSimpleRecvBuffers = false;
1165  if(numSend != totalLocalHaloNodes*testHalo.numStateComponents)
1166  createSimpleSendBuffers = false;
1167  if(numRecv != totalRemoteHaloNodes*testHalo.numStateComponents)
1168  createSimpleRecvBuffers = false;
1169 
1170  testHalo.CreateHaloBuffers();
1171  bool createhalobuffers = true;
1172 
1173  bool packsimplesend = true;
1174  testHalo.PackSimpleSendBuffers(testState);
1175  int numSendBuffers = sendBuffers.size();
1176  for(int iSend = 0;iSend < numSendBuffers;iSend++){
1177  // std::cout << "Send buffer " << iSend << ":" << std::endl;
1178  pcpp::IndexIntervalType &sendExtent(localHaloExtents[iSend]);
1179  std::vector<size_t> sendNodeIds;
1180  partitionExtent.GetFlatIndices(sendExtent,sendNodeIds);
1181  size_t numSendNodes = sendExtent.NNodes();
1182  size_t numValsSend = numSendNodes * testHalo.numStateComponents;
1183  for(size_t iVal = 0;iVal < numValsSend;iVal++){
1184  size_t componentId = iVal/numSendNodes;
1185  size_t sendNodeId = iVal%numSendNodes;
1186  size_t partitionNodeId = sendNodeIds[sendNodeId];
1187  double expectedValue = componentId*nNodes + partitionNodeId;
1188  if(sendBuffers[iSend][iVal] != expectedValue){
1189  packsimplesend = false;
1190  std::cout << "Node (send/par): (" << sendNodeId << "," << partitionNodeId
1191  << ") Comp:" << componentId << " Value (exp/act): ("
1192  << expectedValue << "," << sendBuffers[iSend][iVal] << ")" << std::endl;
1193  }
1194  }
1195  }
1196 
1197 
1198  // Simulate a receive by populating the receive buffers with the send buffers
1199  // correpsonding to the the *opposite* direction.
1200  for(int iDim = 0;iDim < 3;iDim++){
1201  size_t numCopyNodes = localHaloExtents[2*iDim].NNodes();
1202  size_t numVals = testHalo.numStateComponents * numCopyNodes;
1203  size_t numBytes = testHalo.numStateComponents * numCopyNodes * sizeof(double);
1204  std::memcpy(&(recvBuffers[2*iDim][0]),&(sendBuffers[2*iDim+1][0]),numBytes);
1205  std::memcpy(&(recvBuffers[2*iDim+1][0]),&(sendBuffers[2*iDim][0]),numBytes);
1206  }
1207 
1208  // Unpack "received" data into the buffers that hold the data to be
1209  // used by kernels in the halo regions
1210  testHalo.UnpackReceiveBuffers();
1211 
1212  // Check the halo buffers for the proper data to see if unpack succeeded
1213  bool unpackrecvbuffers = true;
1214  std::vector<std::vector<double *> > &haloBuffers(testHalo.HaloBuffers());
1215  for(int iField = 0;iField < testHalo.numStateFields;iField++){
1216  std::vector<double *> &fieldBuffers(haloBuffers[iField]);
1217  int numFieldComponents = testHalo.numFieldComponents[iField];
1218  for(int iDim = 0;iDim < 3;iDim++){
1219  size_t numHaloNodes = localHaloExtents[2*iDim].NNodes();
1220  const double *haloBuffer0(fieldBuffers[2*iDim]);
1221  const double *haloBuffer1(fieldBuffers[2*iDim+1]);
1222  const double *sendBuffer0(&sendBuffers[2*iDim][iField*numHaloNodes]);
1223  const double *sendBuffer1(&sendBuffers[2*iDim+1][iField*numHaloNodes]);
1224  for(int iComponent = 0;iComponent < numFieldComponents;iComponent++){
1225  size_t bufComp = iComponent * numHaloNodes;
1226  for(int iVal = 0;iVal < numHaloNodes;iVal++){
1227  int bufIndex = bufComp + iVal;
1228  if(haloBuffer0[bufIndex] != sendBuffer1[bufIndex] ||
1229  haloBuffer1[bufIndex] != sendBuffer0[bufIndex])
1230  unpackrecvbuffers = false;
1231  }
1232  }
1233  }
1234  }
1235 
1236 
1237  serialUnitResults.UpdateResult("Halo:GlobalExtentsConstructor",constructorworks);
1238  serialUnitResults.UpdateResult("Halo:CreateRemoteHaloExtents",createremotehaloextents);
1239  serialUnitResults.UpdateResult("Halo:CreateLocalHaloExtents",createlocalhaloextents);
1240  serialUnitResults.UpdateResult("Halo:SetRemoteHaloExtents",setremotehaloextents);
1241  serialUnitResults.UpdateResult("Halo:SetLocalHaloExtents",setlocalhaloextents);
1242  serialUnitResults.UpdateResult("Halo:ConfigureData",configuredata);
1243  serialUnitResults.UpdateResult("Halo:CreateSimpleSendBuffers",createSimpleSendBuffers);
1244  serialUnitResults.UpdateResult("Halo:CreateSimpleRecvBuffers",createSimpleRecvBuffers);
1245  serialUnitResults.UpdateResult("Halo:CreateHaloBuffers",createhalobuffers);
1246  serialUnitResults.UpdateResult("Halo:PackSimpleSendBuffers",packsimplesend);
1247  serialUnitResults.UpdateResult("Halo:UnpackReceiveBuffers",unpackrecvbuffers);
1248 
1249 };
1250 
1251 void TestHaloThreaded(ix::test::results &serialUnitResults) {
1252 
1253  int numThreads = 4;
1254 
1255  pcpp::IndexIntervalType globalExtent;
1256  std::vector<size_t> globalSizes(3,101);
1257  globalExtent.InitSimple(globalSizes);
1258 
1259  pcpp::IndexIntervalType partitionExtent;
1260  std::vector<size_t> startIndices(3,5);
1261  std::vector<size_t> partitionSizes(3,8);
1262  partitionExtent.Init(startIndices,partitionSizes);
1263 
1264  size_t numX = 8;
1265  size_t numY = 8;
1266  size_t numZ = 8;
1267  size_t nCells = 343;
1268  size_t nNodes = 512;
1269 
1270  bool constructorworks = true;
1271  simulation::state::halo testHalo(globalExtent,partitionExtent);
1272  // testHalo.ConfigureHalo(globalExtent,partitionExtent);
1273  if(testHalo.GlobalExtent() != globalExtent)
1274  constructorworks = false;
1275  if(testHalo.PartitionExtent() != partitionExtent)
1276  constructorworks = false;
1277 
1278  bool setNumThreadsWorks = true;
1279  testHalo.SetNumThreads(numThreads);
1280  std::vector<pcpp::IndexIntervalType> &threadExtents(testHalo.ThreadExtents());
1281  if(threadExtents.size() != numThreads)
1282  setNumThreadsWorks = false;
1283  if(testHalo.NumThreads() != numThreads)
1284  setNumThreadsWorks = false;
1285  serialUnitResults.UpdateResult("Halo:SetNumThreads",setNumThreadsWorks);
1286 
1287  std::vector<size_t> haloSizes(6,2);
1288  std::vector<pcpp::IndexIntervalType> remoteHaloExtents(testHalo.CreateRemoteHaloExtents(haloSizes));
1289  std::vector<pcpp::IndexIntervalType> localHaloExtents(testHalo.CreateLocalHaloExtents(haloSizes));
1290 
1291  bool createremotehaloextents = true;
1292  bool createlocalhaloextents = true;
1293 
1294  if(remoteHaloExtents.size() != 6)
1295  createremotehaloextents = false;
1296  if(localHaloExtents.size() != 6)
1297  createlocalhaloextents = false;
1298 
1299 
1300  testHalo.SetRemoteHaloExtents(remoteHaloExtents);
1301  testHalo.SetLocalHaloExtents(localHaloExtents);
1302 
1303  simulation::state::base testState;
1304  testState.AddField("time",'s',1,8,"s");
1305  testState.AddField("U",'n',1,8,"stuff");
1306  testState.AddField("V",'n',3,8,"things");
1307  testState.AddField("iblank",'n',1,4,"");
1308  testState.Create(nNodes,nCells);
1309 
1310  std::vector<double> uVector(nNodes,0);
1311  std::vector<double> vVector(nNodes*3,0);
1312  testState.SetFieldBuffer("U",uVector);
1313  testState.SetFieldBuffer("V",vVector);
1314 
1315  for(int iNode = 0;iNode < nNodes;iNode++){
1316  uVector[iNode] = iNode;
1317  vVector[iNode] = uVector[iNode] + nNodes;
1318  vVector[iNode+nNodes] = vVector[iNode] + nNodes;
1319  vVector[iNode+(2*nNodes)] = vVector[iNode+nNodes] + nNodes;
1320  }
1321 
1322  std::vector<int> stateFieldIndices(2,1);
1323  stateFieldIndices[1] = 2;
1324  testState.SetStateFields(stateFieldIndices);
1325 
1326  testHalo.ConfigureData(testState);
1327 
1328  bool createSimpleSendBuffers = true;
1329  bool createSimpleRecvBuffers = true;
1330 
1331  int numSend = testHalo.CreateSimpleSendBuffers();
1332  int numRecv = testHalo.CreateSimpleRecvBuffers();
1333 
1334  std::vector<double *> &sendBuffers(testHalo.SendBuffers());
1335  std::vector<double *> &recvBuffers(testHalo.RecvBuffers());
1336 
1337  testHalo.CreateHaloBuffers();
1338  std::vector<std::vector<double *> > &haloBuffers(testHalo.HaloBuffers());
1339 
1340  std::vector<bool> partDirs(3,true);
1341  partDirs[0] = false;
1342  bool createThreadSendInd = true;
1343  for(int iThread = 0;iThread < numThreads;iThread++){
1344  std::vector<int> numThreadParts;
1345  if(pcpp::util::SimplePartitionInterval(partitionExtent,partDirs,
1346  iThread,numThreads,threadExtents[iThread],numThreadParts)){
1347 
1348  exit(1);
1349  }
1350  std::cout << "Thread(" << iThread << "): ";
1351  threadExtents[iThread].PrettyPrint(std::cout);
1352  std::cout << std::endl;
1353  testHalo.SetThreadExtent(iThread,threadExtents[iThread]);
1354  if(testHalo.CreateThreadSendIndices(iThread))
1355  createThreadSendInd = false;
1356 
1357  // Make sure the buffer fill was sane. Each thread should have "touched"
1358  // the send buffer with a value = -100*(threadId+1)*(sendId+1), make sure
1359  // that's actually the value the send buffer got
1360  int numSendBuffers = sendBuffers.size();
1361  for(int iSend = 0;iSend < numSendBuffers;iSend++){
1362  double *sendBuffer = sendBuffers[iSend]; // grab the send buffer
1363  pcpp::IndexIntervalType &sendExtent(localHaloExtents[iSend]); // send region index interval
1364  size_t numSendNodes = sendExtent.NNodes(); // number of nodes in the send region
1365  // collide the send extent with the thread extent to get the thread-local send extent
1366  pcpp::IndexIntervalType threadSendExtent(sendExtent.Overlap(threadExtents[iThread]));
1367  size_t numThreadSend = threadSendExtent.NNodes(); // number of nodes this thread sends
1368  if(numThreadSend > 0){
1369  // grab local partition node ids that this thread sends
1370  std::vector<size_t> sendNodeIds;
1371  partitionExtent.GetFlatIndices(threadSendExtent,sendNodeIds);
1372  // grab the corresponding send buffer indices
1373  std::vector<size_t> sendBufferIndices;
1374  sendExtent.GetFlatIndices(threadSendExtent,sendBufferIndices);
1375  // loop over the values in the send buffer and check that they're correct
1376  for(size_t iComponent = 0;iComponent < testHalo.numStateComponents;iComponent++){
1377  size_t componentIndex = iComponent*numSendNodes;
1378  for(size_t iThreadNode = 0;iThreadNode < numThreadSend;iThreadNode++){
1379  size_t sendBufferIndex = sendBufferIndices[iThreadNode] + componentIndex;
1380  if(sendBuffer[sendBufferIndex] != -100*(iThread+1)*(iSend+1))
1381  createThreadSendInd = false;
1382  }
1383  }
1384  }
1385  }
1386  }
1387  serialUnitResults.UpdateResult("Halo:CreateThreadSendIndices",createThreadSendInd);
1388 
1389  bool threadedpack = true;
1390  for(int iThread = 0;iThread < numThreads;iThread++){
1391  if(testHalo.PackSimpleSendBuffers(testState,iThread))
1392  threadedpack = false;
1393  }
1394 
1395  int numSendBuffers = sendBuffers.size();
1396  for(int iSend = 0;iSend < numSendBuffers;iSend++){
1397  // std::cout << "Send buffer " << iSend << ":" << std::endl;
1398  pcpp::IndexIntervalType &sendExtent(localHaloExtents[iSend]);
1399  std::vector<size_t> sendNodeIds;
1400  partitionExtent.GetFlatIndices(sendExtent,sendNodeIds);
1401  size_t numSendNodes = sendExtent.NNodes();
1402  size_t numValsSend = numSendNodes * testHalo.numStateComponents;
1403  for(size_t iVal = 0;iVal < numValsSend;iVal++){
1404  size_t componentId = iVal/numSendNodes;
1405  size_t sendNodeId = iVal%numSendNodes;
1406  size_t partitionNodeId = sendNodeIds[sendNodeId];
1407  double expectedValue = componentId*nNodes + partitionNodeId;
1408  if(sendBuffers[iSend][iVal] != expectedValue){
1409  threadedpack = false;
1410  std::cout << "Node (send/par): (" << sendNodeId << "," << partitionNodeId
1411  << ") Comp:" << componentId << " Value (exp/act): ("
1412  << expectedValue << "," << sendBuffers[iSend][iVal] << ")" << std::endl;
1413  }
1414  }
1415  }
1416  serialUnitResults.UpdateResult("Halo:ThreadedPack",threadedpack);
1417 
1418 
1419  bool createThreadRecvInd = true;
1420  for(int iThread = 0;iThread < numThreads;iThread++){
1421  if(testHalo.CreateThreadRecvIndices(iThread))
1422  createThreadRecvInd = false;
1423 
1424 
1425  // Make sure the buffer fills were sane. Each thread should have "touched"
1426  // the recv buffer with a value = -100*(threadId+1)+(sendId+1), make sure
1427  // that's actually the value the recv buffer got
1428  int numRecvBuffers = recvBuffers.size();
1429  for(int iRecv = 0;iRecv < numRecvBuffers;iRecv++){
1430  std::vector<double *> &remoteHaloBuffers(haloBuffers[iRecv]);
1431  double *recvBuffer = recvBuffers[iRecv]; // grab the send buffer
1432  pcpp::IndexIntervalType &haloExtent(remoteHaloExtents[iRecv]); // send region index interval
1433  size_t numRecvNodes = haloExtent.NNodes(); // number of nodes in the halo region
1434  // collide the halo extent with the thread extent to get the thread-local recv extent
1435  pcpp::IndexIntervalType threadRecvExtent(haloExtent.Overlap(threadExtents[iThread]));
1436  size_t numThreadRecv = threadRecvExtent.NNodes(); // number of nodes this thread receives
1437  if(numThreadRecv > 0){
1438  std::cout << "NumThreadRecv: " << numThreadRecv << std::endl;
1439  // grab local partition node ids that this thread receives
1440  // std::vector<size_t> recvNodeIds;
1441  // partitionExtent.GetFlatIndices(threadRecvExtent,recvNodeIds);
1442  // grab the recv buffer indices
1443  int componentId = 0;
1444  std::vector<size_t> recvBufferIndices;
1445  haloExtent.GetFlatIndices(threadRecvExtent,recvBufferIndices);
1446  // loop over the values in the recv buffer and check that they're correct
1447  for(int iField = 0;iField < testHalo.numStateFields;iField++){
1448  int numFieldComponents = testHalo.numFieldComponents[iField];
1449  double *haloBuffer = remoteHaloBuffers[iField];
1450  for(int iComponent = 0;iComponent < numFieldComponents;iComponent++){
1451  size_t haloComponentIndex = iComponent*numRecvNodes;
1452  size_t recvComponentIndex = numRecvNodes*componentId++;
1453  for(size_t iThreadNode = 0;iThreadNode < numThreadRecv;iThreadNode++){
1454  size_t recvBufferIndex = recvBufferIndices[iThreadNode] + recvComponentIndex;
1455  size_t haloBufferIndex = recvBufferIndices[iThreadNode] + haloComponentIndex;
1456  if(recvBuffer[recvBufferIndex] != -100*(iThread+1)+(iRecv+1))
1457  createThreadRecvInd = false;
1458  if(haloBuffer[haloBufferIndex] != -100*(iThread+1)+(iRecv+1))
1459  createThreadRecvInd = false;
1460  }
1461  }
1462  }
1463  }
1464  }
1465  }
1466  serialUnitResults.UpdateResult("Halo:CreateThreadRecvIndices",createThreadRecvInd);
1467 
1468  // Simulate a receive by populating the receive buffers with the send buffers
1469  // correpsonding to the the *opposite* direction.
1470  for(int iDim = 0;iDim < 3;iDim++){
1471  size_t numCopyNodes = localHaloExtents[2*iDim].NNodes();
1472  size_t numVals = testHalo.numStateComponents * numCopyNodes;
1473  size_t numBytes = testHalo.numStateComponents * numCopyNodes * sizeof(double);
1474  std::memcpy(&(recvBuffers[2*iDim][0]),&(sendBuffers[2*iDim+1][0]),numBytes);
1475  std::memcpy(&(recvBuffers[2*iDim+1][0]),&(sendBuffers[2*iDim][0]),numBytes);
1476  }
1477 
1478  // Unpack "received" data into the buffers that hold the data to be
1479  // used by kernels in the halo regions
1480  bool unpackthreaded = true;
1481  for(int iThread = 0;iThread < numThreads;iThread++)
1482  if(testHalo.UnpackSimpleRecvBuffers(iThread))
1483  unpackthreaded = false;
1484 
1485  // Check the halo buffers for the proper data to see if unpack succeeded
1486  for(int iField = 0;iField < testHalo.numStateFields;iField++){
1487  std::vector<double *> &fieldBuffers(haloBuffers[iField]);
1488  int numFieldComponents = testHalo.numFieldComponents[iField];
1489  for(int iDim = 0;iDim < 3;iDim++){
1490  size_t numHaloNodes = localHaloExtents[2*iDim].NNodes();
1491  const double *haloBuffer0(fieldBuffers[2*iDim]);
1492  const double *haloBuffer1(fieldBuffers[2*iDim+1]);
1493  const double *sendBuffer0(&sendBuffers[2*iDim][iField*numHaloNodes]);
1494  const double *sendBuffer1(&sendBuffers[2*iDim+1][iField*numHaloNodes]);
1495  for(int iComponent = 0;iComponent < numFieldComponents;iComponent++){
1496  size_t bufComp = iComponent * numHaloNodes;
1497  for(int iVal = 0;iVal < numHaloNodes;iVal++){
1498  int bufIndex = bufComp + iVal;
1499  if(haloBuffer0[bufIndex] != sendBuffer1[bufIndex] ||
1500  haloBuffer1[bufIndex] != sendBuffer0[bufIndex])
1501  unpackthreaded = false;
1502  }
1503  }
1504  }
1505  }
1506  serialUnitResults.UpdateResult("Halo:ThreadedUnPack",unpackthreaded);
1507 
1508 
1509 // serialUnitResults.UpdateResult("Halo:GlobalExtentsConstructor",constructorworks);
1510 // serialUnitResults.UpdateResult("Halo:CreateRemoteHaloExtents",createremotehaloextents);
1511 // serialUnitResults.UpdateResult("Halo:CreateLocalHaloExtents",createlocalhaloextents);
1512 // serialUnitResults.UpdateResult("Halo:SetRemoteHaloExtents",setremotehaloextents);
1513 // serialUnitResults.UpdateResult("Halo:SetLocalHaloExtents",setlocalhaloextents);
1514 // serialUnitResults.UpdateResult("Halo:ConfigureData",configuredata);
1515 // serialUnitResults.UpdateResult("Halo:CreateSimpleSendBuffers",createSimpleSendBuffers);
1516 // serialUnitResults.UpdateResult("Halo:CreateSimpleRecvBuffers",createSimpleRecvBuffers);
1517 // serialUnitResults.UpdateResult("Halo:CreateHaloBuffers",createhalobuffers);
1518 // serialUnitResults.UpdateResult("Halo:PackSimpleSendBuffers",packsimplesend);
1519 // serialUnitResults.UpdateResult("Halo:UnpackReceiveBuffers",unpackrecvbuffers);
1520 
1521 };
1522 
pcpp::comm::paralleltopologyinfo::cartDecompDirections
std::vector< int > cartDecompDirections
Definition: PCPPCommUtil.H:22
simulation::state::halo::SetNumThreads
void SetNumThreads(int numThreadsIn)
Definition: State.H:531
simulation::state::halo::CreateThreadRecvIndices
int CreateThreadRecvIndices(int threadId)
Create receive buffers for state data from neighboring processors.
Definition: State.C:826
simulation::state::halo::SendBuffers
std::vector< double * > & SendBuffers()
Definition: State.H:611
simulation::state::halo::HaloBuffers
std::vector< std::vector< double * > > & HaloBuffers()
Definition: State.H:609
simulation::state::halo::PostSimpleReceives
int PostSimpleReceives(std::vector< int > &sourceRanks, CommunicatorType &inComm)
Definition: State.C:1456
TestHaloPeriodic
void TestHaloPeriodic(ix::test::results &parallelUnitResults, pcpp::CommunicatorType &testComm)
Definition: TestHalo.C:448
simulation::state::halo::CreateHaloBuffers
void CreateHaloBuffers()
Definition: State.C:238
pcpp::util::SimplePartitionInterval
int SimplePartitionInterval(const pcpp::IndexIntervalType &inInterval, std::vector< bool > partDirection, int partID, int numPart, pcpp::IndexIntervalType &outInterval, std::vector< int > &numPartitions)
Multi-dimensional interval partitioning (non-MPI)
Definition: PCPPIntervalUtils.C:151
simulation::state::halo::RemoteHaloExtents
std::vector< pcpp::IndexIntervalType > & RemoteHaloExtents()
Definition: State.H:625
ix::util::sizeextent::GetFlatIndices
void GetFlatIndices(const sizeextent &extent, ContainerType &indices) const
Definition: IndexUtil.H:302
simulation::state::halo::numFieldComponents
std::vector< int > numFieldComponents
Definition: State.H:429
simulation::state::base::SetStateFields
int SetStateFields(const std::vector< int > &inFieldIndices)
Definition: State.H:310
ix::comm::CommunicatorObject::CartNeighbors
int CartNeighbors(std::vector< int > &neighborRanks)
Definition: COMM.C:173
simulation::state::halo::CompleteSimpleReceives
int CompleteSimpleReceives(CommunicatorType &inComm)
Definition: State.C:1553
simulation::state::halo::ThreadExtents
std::vector< pcpp::IndexIntervalType > & ThreadExtents()
Definition: State.H:621
simulation::state::halo::UnpackReceiveBuffers
int UnpackReceiveBuffers()
Definition: State.C:926
TestHaloThreaded
void TestHaloThreaded(ix::test::results &serialUnitResults)
Definition: TestHalo.C:1251
simulation::state::halo::CreateSimpleRecvBuffers
int CreateSimpleRecvBuffers()
Create receive buffers for state data from neighboring processors.
Definition: State.C:554
ix::util::RenewStream
void RenewStream(std::ostringstream &outStream)
Definition: primitive_utilities.H:638
pcpp::comm::CheckResult
bool CheckResult(bool &localResult, pcpp::CommunicatorType &testComm)
Definition: PCPPCommUtil.C:176
simulation::state::halo::PartitionExtent
pcpp::IndexIntervalType & PartitionExtent()
Definition: State.H:624
ix::comm::CommunicatorObject::SetErr
int SetErr(int errin)
Definition: COMM.H:110
pcpp::report::CartesianSetup
void CartesianSetup(std::ostream &outStream, const std::vector< int > &cartCoords, const std::vector< int > &cartDims)
Definition: PCPPReport.C:199
pcpp::field::dataset::Create
virtual size_t Create(size_t number_of_nodes=0, size_t number_of_cells=0)
Definition: PCPPFieldData.H:578
simulation::state::halo::SetNeighbors
void SetNeighbors(const std::vector< bool > &inNeighbors)
Definition: State.H:604
ix::util::sizeextent::NNodes
size_t NNodes() const
Definition: IndexUtil.H:254
simulation::state::halo::CreateRemoteHaloExtents
std::vector< pcpp::IndexIntervalType > CreateRemoteHaloExtents(const pcpp::IndexIntervalType &globalExtent, const pcpp::IndexIntervalType &partitionExtent, std::vector< size_t > &haloSizes)
Definition: State.C:78
simulation::state::halo::SetThreadExtent
void SetThreadExtent(int myThreadId, pcpp::IndexIntervalType threadInterval)
Definition: State.H:514
simulation::state::halo::ConfigureData
void ConfigureData(const state::base &inState)
Definition: State.C:1513
ix::test::results
Encapsulating class for collections of test results.
Definition: Testing.H:18
TestHaloParallel
void TestHaloParallel(ix::test::results &parallelUnitResults, pcpp::CommunicatorType &testComm)
Definition: TestHalo.C:9
pcpp::comm::paralleltopologyinfo::isPeriodic
std::vector< int > isPeriodic
Definition: PCPPCommUtil.H:24
pcpp::io::Everyone
void Everyone(const std::string &outString, std::ostream &outStream, fixtures::CommunicatorType &comm)
Definition: PCPPIO.C:51
ix::comm::CommunicatorObject::Check
int Check(comm::Ops op=comm::MAXOP)
Definition: COMM.C:355
simulation::state::halo::SetLocalHaloExtents
void SetLocalHaloExtents(const std::vector< pcpp::IndexIntervalType > haloExtents)
Definition: State.H:545
ix::comm::CommunicatorObject
Main encapsulation of MPI.
Definition: COMM.H:62
simulation::state::halo::UnpackSimpleRecvBuffers
int UnpackSimpleRecvBuffers(int threadId)
Definition: State.C:1009
Testing.H
Testing constructs for unit testing.
simulation::state::halo::StateFieldIndices
std::vector< int > & StateFieldIndices()
Definition: State.H:622
simulation::state::halo::CreateThreadSendIndices
int CreateThreadSendIndices(int threadId)
Creates send buffers for each whole local halo extent.
Definition: State.C:397
ix::util::sizeextent::Overlap
void Overlap(const sizeextent &inextent, sizeextent &outextent) const
Definition: IndexUtil.H:324
simulation::state::halo::SimpleSend
int SimpleSend(std::vector< int > &neighborRanks, CommunicatorType &inComm)
Definition: State.C:1115
pcpp::util::PartitionCartesianExtent
int PartitionCartesianExtent(const pcpp::IndexIntervalType &globalExtent, const std::vector< int > &cartDims, const std::vector< int > &cartCoords, pcpp::IndexIntervalType &partExtent, std::ostream &messageStream)
Get local sub-interval of an n-dimensional integer interval.
Definition: PCPPIntervalUtils.C:317
simulation::state::halo::RecvBuffers
std::vector< double * > & RecvBuffers()
Definition: State.H:607
simulation::state::halo::SetRemoteHaloExtents
void SetRemoteHaloExtents(const std::vector< pcpp::IndexIntervalType > haloExtents)
Definition: State.H:485
ix::util::sizeextent::Init
void Init(const ContainerType &inflatextent)
Definition: IndexUtil.H:133
ix::test::results::UpdateResult
void UpdateResult(const std::string &name, const ValueType &result)
Updates an existing test result.
Definition: Testing.H:55
ix::util::sizeextent::PrettyPrint
std::ostream & PrettyPrint(std::ostream &outStream) const
Definition: IndexUtil.C:6
pcpp::comm::SetupCartesianTopology
int SetupCartesianTopology(pcpp::CommunicatorType &parallelCommunicator, pcpp::ParallelTopologyInfoType &topologyInfo, pcpp::CommunicatorType &topoCommunicator, std::ostream &messageStream)
Sets up a communicator with Cartesian topology.
Definition: PCPPCommUtil.C:48
ix::util::sizeextent
Simple Block Structured Mesh object.
Definition: IndexUtil.H:21
pcpp::field::dataset::AddField
void AddField(const std::string &name, char loc, unsigned int ncomp, unsigned int dsize, const std::string &unit)
Definition: PCPPFieldData.H:869
simulation::state::halo::CreateSimpleSendBuffers
int CreateSimpleSendBuffers()
Creates send buffers for each whole local halo extent.
Definition: State.C:347
simulation::state::halo::PackSimpleSendBuffers
int PackSimpleSendBuffers(const state::base &inX, int threadId)
Packs a send buffer with all components of state fields.
Definition: State.C:1301
pcpp::field::dataset::SetFieldBuffer
void SetFieldBuffer(const std::string &name, void *buf)
Definition: PCPPFieldData.H:716
simulation::state::halo::CreateLocalHaloExtents
std::vector< pcpp::IndexIntervalType > CreateLocalHaloExtents(const pcpp::IndexIntervalType &globalExtent, const pcpp::IndexIntervalType &partitionExtent, std::vector< size_t > &haloSizes)
Definition: State.C:155
TestHaloBasic
void TestHaloBasic(ix::test::results &serialUnitResults)
Definition: TestHalo.C:1016
ix::comm::CommunicatorObject::CartDimensions
std::vector< int > & CartDimensions()
Definition: COMM.H:118
ix::util::sizeextent::InitSimple
void InitSimple(const ContainerType &inSize)
Definition: IndexUtil.H:169
simulation::state::halo::GlobalExtent
pcpp::IndexIntervalType & GlobalExtent()
Definition: State.H:623
simulation::state::halo::LocalHaloExtents
std::vector< pcpp::IndexIntervalType > & LocalHaloExtents()
Definition: State.H:626
ix::comm::CommunicatorObject::CartCoordinates
std::vector< int > & CartCoordinates()
Definition: COMM.H:117