Commit 783b5736 authored by mattijs's avatar mattijs
Browse files

correct non-blocking

parent b52bae62
......@@ -84,16 +84,32 @@ void Foam::mapDistribute::distribute
// Receive sub field from neighbour
for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
if
(
domain != Pstream::myProcNo()
&& constructMap[domain].size() > 0
)
const labelList& map = constructMap[domain];
if (domain != Pstream::myProcNo() && map.size() > 0)
{
IPstream fromNbr(Pstream::blocking, domain);
List<T> subField(fromNbr);
const labelList& map = constructMap[domain];
if (subField.size() != map.size())
{
FatalErrorIn
(
"template<class T>\n"
"void mapDistribute::distribute\n"
"(\n"
" const Pstream::commsTypes commsType,\n"
" const List<labelPair>& schedule,\n"
" const label constructSize,\n"
" const labelListList& subMap,\n"
" const labelListList& constructMap,\n"
" List<T>& field\n"
")\n"
) << "Expected from processor " << domain
<< " " << map.size() << " but received "
<< subField.size() << " elements."
<< abort(FatalError);
}
forAll(map, i)
{
......@@ -126,6 +142,7 @@ void Foam::mapDistribute::distribute
newField[map[i]] = subField[i];
}
// Schedule will already have pruned 0-sized comms
forAll(schedule, i)
{
const labelPair& twoProcs = schedule[i];
......@@ -154,6 +171,26 @@ void Foam::mapDistribute::distribute
const labelList& map = constructMap[sendProc];
if (subField.size() != map.size())
{
FatalErrorIn
(
"template<class T>\n"
"void mapDistribute::distribute\n"
"(\n"
" const Pstream::commsTypes commsType,\n"
" const List<labelPair>& schedule,\n"
" const label constructSize,\n"
" const labelListList& subMap,\n"
" const labelListList& constructMap,\n"
" List<T>& field\n"
")\n"
) << "Expected from processor " << sendProc
<< " " << map.size() << " but received "
<< subField.size() << " elements."
<< abort(FatalError);
}
forAll(map, i)
{
newField[map[i]] = subField[i];
......@@ -164,65 +201,134 @@ void Foam::mapDistribute::distribute
}
else if (commsType == Pstream::nonBlocking)
{
List<T> newField(constructSize);
// Subset myself
const labelList& mySubMap = subMap[Pstream::myProcNo()];
List<T> subField(mySubMap.size());
forAll(mySubMap, i)
if (!contiguous<T>())
{
subField[i] = field[mySubMap[i]];
FatalErrorIn
(
"template<class T>\n"
"void mapDistribute::distribute\n"
"(\n"
" const Pstream::commsTypes commsType,\n"
" const List<labelPair>& schedule,\n"
" const label constructSize,\n"
" const labelListList& subMap,\n"
" const labelListList& constructMap,\n"
" List<T>& field\n"
")\n"
) << "Non-blocking only supported for contiguous data."
<< exit(FatalError);
}
// Receive sub field from myself (subField)
const labelList& map = constructMap[Pstream::myProcNo()];
// Set up sends to neighbours
forAll(map, i)
{
newField[map[i]] = subField[i];
}
List<List<T > > sendFields(Pstream::nProcs());
// Send sub field to neighbour
for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
const labelList& map = subMap[domain];
if (domain != Pstream::myProcNo() && map.size() > 0)
{
List<T> subField(map.size());
List<T>& subField = sendFields[domain];
subField.setSize(map.size());
forAll(map, i)
{
subField[i] = field[map[i]];
}
OPstream toNbr(Pstream::nonBlocking, domain);
toNbr << subField;
OPstream::write
(
Pstream::nonBlocking,
domain,
reinterpret_cast<const char*>(subField.begin()),
subField.size()
);
}
}
// Set up receives from neighbours
List<List<T > > recvFields(Pstream::nProcs());
// Receive sub field from neighbour
for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
const labelList& map = constructMap[domain];
if (domain != Pstream::myProcNo() && map.size() > 0)
{
IPstream fromNbr(Pstream::nonBlocking, domain);
List<T> subField(fromNbr);
recvFields[domain].setSize(map.size());
IPstream::read
(
Pstream::nonBlocking,
domain,
reinterpret_cast<char*>(recvFields[domain].begin()),
recvFields[domain].size()
);
}
}
// Combine bits. Note that can reuse field storage
// Subset myself
const labelList& mySubMap = subMap[Pstream::myProcNo()];
List<T> subField(mySubMap.size());
forAll(mySubMap, i)
{
subField[i] = field[mySubMap[i]];
}
field.setSize(constructSize);
// Receive sub field from myself (subField)
const labelList& map = constructMap[Pstream::myProcNo()];
forAll(map, i)
{
field[map[i]] = subField[i];
}
// Wait for all to finish
OPstream::waitRequests();
IPstream::waitRequests();
// Collect neighbour fields
for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
const labelList& map = constructMap[domain];
if (domain != Pstream::myProcNo() && map.size() > 0)
{
if (recvFields[domain].size() != map.size())
{
FatalErrorIn
(
"template<class T>\n"
"void mapDistribute::distribute\n"
"(\n"
" const Pstream::commsTypes commsType,\n"
" const List<labelPair>& schedule,\n"
" const label constructSize,\n"
" const labelListList& subMap,\n"
" const labelListList& constructMap,\n"
" List<T>& field\n"
")\n"
) << "Expected from processor " << domain
<< " " << map.size() << " but received "
<< recvFields[domain].size() << " elements."
<< abort(FatalError);
}
forAll(map, i)
{
newField[map[i]] = subField[i];
field[map[i]] = recvFields[domain][i];
}
}
}
OPstream::waitRequests();
IPstream::waitRequests();
field.transfer(newField);
}
else
{
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment