correct non-blocking

src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C

@@ -84,16 +84,32 @@ void Foam::mapDistribute::distribute
         // Receive sub field from neighbour
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
-            if
-            (
-                domain != Pstream::myProcNo()
-             && constructMap[domain].size() > 0
-            )
+            const labelList& map = constructMap[domain];
+
+            if (domain != Pstream::myProcNo() && map.size() > 0)
             {
                 IPstream fromNbr(Pstream::blocking, domain);
                 List<T> subField(fromNbr);
 
-                const labelList& map = constructMap[domain];
+                if (subField.size() != map.size())
+                {
+                    FatalErrorIn
+                    (
+                        "template<class T>\n"
+                        "void mapDistribute::distribute\n"
+                        "(\n"
+                        "    const Pstream::commsTypes commsType,\n"
+                        "    const List<labelPair>& schedule,\n"
+                        "    const label constructSize,\n"
+                        "    const labelListList& subMap,\n"
+                        "    const labelListList& constructMap,\n"
+                        "    List<T>& field\n"
+                        ")\n"
+                    )   << "Expected from processor " << domain
+                        << " " << map.size() << " but received "
+                        << subField.size() << " elements."
+                        << abort(FatalError);
+                }
 
                 forAll(map, i)
                 {
@@ -126,6 +142,7 @@ void Foam::mapDistribute::distribute
             newField[map[i]] = subField[i];
         }
 
+        // Schedule will already have pruned 0-sized comms
         forAll(schedule, i)
         {
             const labelPair& twoProcs = schedule[i];
@@ -154,6 +171,26 @@ void Foam::mapDistribute::distribute
 
                 const labelList& map = constructMap[sendProc];
 
+                if (subField.size() != map.size())
+                {
+                    FatalErrorIn
+                    (
+                        "template<class T>\n"
+                        "void mapDistribute::distribute\n"
+                        "(\n"
+                        "    const Pstream::commsTypes commsType,\n"
+                        "    const List<labelPair>& schedule,\n"
+                        "    const label constructSize,\n"
+                        "    const labelListList& subMap,\n"
+                        "    const labelListList& constructMap,\n"
+                        "    List<T>& field\n"
+                        ")\n"
+                    )   << "Expected from processor " << sendProc
+                        << " " << map.size() << " but received "
+                        << subField.size() << " elements."
+                        << abort(FatalError);
+                }
+
                 forAll(map, i)
                 {
                     newField[map[i]] = subField[i];
@@ -164,65 +201,134 @@ void Foam::mapDistribute::distribute
     }
     else if (commsType == Pstream::nonBlocking)
     {
-        List<T> newField(constructSize);
-
-        // Subset myself
-        const labelList& mySubMap = subMap[Pstream::myProcNo()];
-
-        List<T> subField(mySubMap.size());
-        forAll(mySubMap, i)
+        if (!contiguous<T>())
         {
-            subField[i] = field[mySubMap[i]];
+            FatalErrorIn
+            (
+                "template<class T>\n"
+                "void mapDistribute::distribute\n"
+                "(\n"
+                "    const Pstream::commsTypes commsType,\n"
+                "    const List<labelPair>& schedule,\n"
+                "    const label constructSize,\n"
+                "    const labelListList& subMap,\n"
+                "    const labelListList& constructMap,\n"
+                "    List<T>& field\n"
+                ")\n"
+            )   << "Non-blocking only supported for contiguous data."
+                << exit(FatalError);
         }
 
-        // Receive sub field from myself (subField)
-        const labelList& map = constructMap[Pstream::myProcNo()];
+        // Set up sends to neighbours
 
-        forAll(map, i)
-        {
-            newField[map[i]] = subField[i];
-        }
+        List<List<T > > sendFields(Pstream::nProcs());
 
-        // Send sub field to neighbour
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
             const labelList& map = subMap[domain];
 
             if (domain != Pstream::myProcNo() && map.size() > 0)
             {
-
-                List<T> subField(map.size());
+                List<T>& subField = sendFields[domain];
+                subField.setSize(map.size());
                 forAll(map, i)
                 {
                     subField[i] = field[map[i]];
                 }
 
-                OPstream toNbr(Pstream::nonBlocking, domain);
-                toNbr << subField;
+                OPstream::write
+                (
+                    Pstream::nonBlocking,
+                    domain,
+                    reinterpret_cast<const char*>(subField.begin()),
+                    subField.size()
+                );
             }
         }
 
+        // Set up receives from neighbours
+
+        List<List<T > > recvFields(Pstream::nProcs());
 
-        // Receive sub field from neighbour
         for (label domain = 0; domain < Pstream::nProcs(); domain++)
         {
             const labelList& map = constructMap[domain];
 
             if (domain != Pstream::myProcNo() && map.size() > 0)
             {
-                IPstream fromNbr(Pstream::nonBlocking, domain);
-                List<T> subField(fromNbr);
+                recvFields[domain].setSize(map.size());
+                IPstream::read
+                (
+                    Pstream::nonBlocking,
+                    domain,
+                    reinterpret_cast<char*>(recvFields[domain].begin()),
+                    recvFields[domain].size()
+                );
+            }
+        }
+
+
+        // Combine bits. Note that can reuse field storage
+
+        // Subset myself
+        const labelList& mySubMap = subMap[Pstream::myProcNo()];
+
+        List<T> subField(mySubMap.size());
+        forAll(mySubMap, i)
+        {
+            subField[i] = field[mySubMap[i]];
+        }
+
+        field.setSize(constructSize);
+
+        // Receive sub field from myself (subField)
+        const labelList& map = constructMap[Pstream::myProcNo()];
+
+        forAll(map, i)
+        {
+            field[map[i]] = subField[i];
+        }
+
+
+        // Wait for all to finish
+
+        OPstream::waitRequests();
+        IPstream::waitRequests();
+        
+        // Collect neighbour fields
+
+        for (label domain = 0; domain < Pstream::nProcs(); domain++)
+        {
+            const labelList& map = constructMap[domain];
+
+            if (domain != Pstream::myProcNo() && map.size() > 0)
+            {
+                if (recvFields[domain].size() != map.size())
+                {
+                    FatalErrorIn
+                    (
+                        "template<class T>\n"
+                        "void mapDistribute::distribute\n"
+                        "(\n"
+                        "    const Pstream::commsTypes commsType,\n"
+                        "    const List<labelPair>& schedule,\n"
+                        "    const label constructSize,\n"
+                        "    const labelListList& subMap,\n"
+                        "    const labelListList& constructMap,\n"
+                        "    List<T>& field\n"
+                        ")\n"
+                    )   << "Expected from processor " << domain
+                        << " " << map.size() << " but received "
+                        << recvFields[domain].size() << " elements."
+                        << abort(FatalError);
+                }
 
                 forAll(map, i)
                 {
-                    newField[map[i]] = subField[i];
+                    field[map[i]] = recvFields[domain][i];
                 }
             }
         }
-        OPstream::waitRequests();
-        IPstream::waitRequests();
-
-        field.transfer(newField);
     }
     else
     {