From 7f43ad47d48cbe758cfba59b6b96d1c89b296ab0 Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Wed, 4 May 2022 18:39:59 +0200
Subject: [PATCH] ENH: mapDistributeBase compaction based on a subset of
elements (#2436)
- now have both compactData(),compactLocalData(), compactRemoteData()
depending on where the compaction information is actually known.
The compactData() performs a consistent union of local and remote
values, which eliminates the danger of mapping to non-existent
locations but does require a double communication to setup.
Typically needed for point maps (for example).
The compactLocalData() and compactRemoteData() work on the
assumption that the source or target values are sufficent for
creating unique compact maps.
Can be used, for example, when compacting cell maps since there is
no possibility of a source cell being represented on different
target processors (ie, each cell is unique and only occurs once).
The existing compact() is equivalent to compactRemoteData()
and is now simply a redirect.
- use bitSet for defining compaction, but the existing compact()
continues to use a boolList (for code compatibility).
BUG: compaction in non-parallel mode didn't compact anything.
STYLE: compact ascii output for procAddressing
---
.../mapDistribute/mapDistributeBase.C | 430 +---------
.../mapDistribute/mapDistributeBase.H | 288 ++++++-
.../mapDistribute/mapDistributeBaseIO.C | 44 +-
.../mapDistribute/mapDistributeBaseSubset.C | 810 ++++++++++++++++++
.../mapDistribute/mapDistributePolyMeshIO.C | 3 +
5 files changed, 1136 insertions(+), 439 deletions(-)
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C
index a3fb7f23559..e6b7537cc78 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C
@@ -944,37 +944,29 @@ Foam::mapDistributeBase::mapDistributeBase
Pstream::exchangeSizes(subMap_, recvSizes, comm_);
// Determine order of receiving
- labelListList constructMap(nProcs);
+ constructSize_ = 0;
+ constructMap_.resize(nProcs);
+
- // My local segments first
- label nLocal = recvSizes[myRank];
+ // My data first
{
- labelList& myMap = constructMap[myRank];
- myMap.setSize(nLocal);
- forAll(myMap, i)
- {
- myMap[i] = i;
- }
+ const label len = recvSizes[myRank];
+
+ constructMap_[myRank] = identity(len, constructSize_);
+ constructSize_ += len;
}
- label segmenti = nLocal;
- forAll(constructMap, proci)
+ // What the other processors are sending to me
+ forAll(constructMap_, proci)
{
if (proci != myRank)
{
- // What i need to receive is what other processor is sending to me.
- label nRecv = recvSizes[proci];
- constructMap[proci].setSize(nRecv);
+ const label len = recvSizes[proci];
- for (label i = 0; i < nRecv; i++)
- {
- constructMap[proci][i] = segmenti++;
- }
+ constructMap_[proci] = identity(len, constructSize_);
+ constructSize_ += len;
}
}
-
- constructSize_ = segmenti;
- constructMap_.transfer(constructMap);
}
@@ -1062,402 +1054,6 @@ Foam::label Foam::mapDistributeBase::renumber
}
-void Foam::mapDistributeBase::compact
-(
- const boolList& elemIsUsed,
- const int tag
-)
-{
- const label myRank = Pstream::myProcNo(comm_);
- const label nProcs = Pstream::nProcs(comm_);
-
- // 1. send back to sender. Have sender delete the corresponding element
- // from the submap and do the same to the constructMap locally
- // (and in same order).
-
- // Send elemIsUsed field to neighbour. Use nonblocking code from
- // mapDistributeBase but in reverse order.
- if (Pstream::parRun())
- {
- label startOfRequests = Pstream::nRequests();
-
- // Set up receives from neighbours
-
- List<boolList> recvFields(nProcs);
-
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = subMap_[domain];
-
- if (domain != myRank && map.size())
- {
- recvFields[domain].setSize(map.size());
- IPstream::read
- (
- Pstream::commsTypes::nonBlocking,
- domain,
- recvFields[domain].data_bytes(),
- recvFields[domain].size_bytes(),
- tag,
- comm_
- );
- }
- }
-
-
- List<boolList> sendFields(nProcs);
-
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = constructMap_[domain];
-
- if (domain != myRank && map.size())
- {
- boolList& subField = sendFields[domain];
- subField.setSize(map.size());
- forAll(map, i)
- {
- subField[i] = accessAndFlip
- (
- elemIsUsed,
- map[i],
- constructHasFlip_,
- identityOp() // Do not flip elemIsUsed value
- );
- }
-
- OPstream::write
- (
- Pstream::commsTypes::nonBlocking,
- domain,
- subField.cdata_bytes(),
- subField.size_bytes(),
- tag,
- comm_
- );
- }
- }
-
-
-
- // Set up 'send' to myself - write directly into recvFields
-
- {
- const labelList& map = constructMap_[myRank];
-
- recvFields[myRank].setSize(map.size());
- forAll(map, i)
- {
- recvFields[myRank][i] = accessAndFlip
- (
- elemIsUsed,
- map[i],
- constructHasFlip_,
- identityOp() // Do not flip elemIsUsed value
- );
- }
- }
-
-
- // Wait for all to finish
-
- Pstream::waitRequests(startOfRequests);
-
-
- // Compact out all submap entries that are referring to unused elements
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = subMap_[domain];
-
- labelList newMap(map.size());
- label newI = 0;
-
- forAll(map, i)
- {
- if (recvFields[domain][i])
- {
- // So element is used on destination side
- newMap[newI++] = map[i];
- }
- }
- if (newI < map.size())
- {
- newMap.setSize(newI);
- subMap_[domain].transfer(newMap);
- }
- }
- }
-
-
- // 2. remove from construct map - since end-result (element in elemIsUsed)
- // not used.
-
- label maxConstructIndex = -1;
-
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = constructMap_[domain];
-
- labelList newMap(map.size());
- label newI = 0;
-
- forAll(map, i)
- {
- label destinationI = map[i];
- if (constructHasFlip_)
- {
- destinationI = mag(destinationI)-1;
- }
-
- // Is element is used on destination side
- if (elemIsUsed[destinationI])
- {
- maxConstructIndex = max(maxConstructIndex, destinationI);
-
- newMap[newI++] = map[i];
- }
- }
- if (newI < map.size())
- {
- newMap.setSize(newI);
- constructMap_[domain].transfer(newMap);
- }
- }
-
- constructSize_ = maxConstructIndex+1;
-
- // Clear the schedule (note:not necessary if nothing changed)
- schedulePtr_.clear();
-}
-
-
-void Foam::mapDistributeBase::compact
-(
- const boolList& elemIsUsed,
- const label localSize, // max index for subMap
- labelList& oldToNewSub,
- labelList& oldToNewConstruct,
- const int tag
-)
-{
- const label myRank = Pstream::myProcNo(comm_);
- const label nProcs = Pstream::nProcs(comm_);
-
- // 1. send back to sender. Have sender delete the corresponding element
- // from the submap and do the same to the constructMap locally
- // (and in same order).
-
- // Send elemIsUsed field to neighbour. Use nonblocking code from
- // mapDistributeBase but in reverse order.
- if (Pstream::parRun())
- {
- label startOfRequests = Pstream::nRequests();
-
- // Set up receives from neighbours
-
- List<boolList> recvFields(nProcs);
-
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = subMap_[domain];
-
- if (domain != myRank && map.size())
- {
- recvFields[domain].setSize(map.size());
- IPstream::read
- (
- Pstream::commsTypes::nonBlocking,
- domain,
- recvFields[domain].data_bytes(),
- recvFields[domain].size_bytes(),
- tag,
- comm_
- );
- }
- }
-
-
- List<boolList> sendFields(nProcs);
-
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = constructMap_[domain];
-
- if (domain != myRank && map.size())
- {
- boolList& subField = sendFields[domain];
- subField.setSize(map.size());
- forAll(map, i)
- {
- label index = map[i];
- if (constructHasFlip_)
- {
- index = mag(index)-1;
- }
- subField[i] = elemIsUsed[index];
- }
-
- OPstream::write
- (
- Pstream::commsTypes::nonBlocking,
- domain,
- subField.cdata_bytes(),
- subField.size_bytes(),
- tag,
- comm_
- );
- }
- }
-
-
-
- // Set up 'send' to myself - write directly into recvFields
-
- {
- const labelList& map = constructMap_[myRank];
-
- recvFields[myRank].setSize(map.size());
- forAll(map, i)
- {
- label index = map[i];
- if (constructHasFlip_)
- {
- index = mag(index)-1;
- }
- recvFields[myRank][i] = elemIsUsed[index];
- }
- }
-
-
- // Wait for all to finish
-
- Pstream::waitRequests(startOfRequests);
-
-
-
-
- // Work out which elements on the sending side are needed
- {
- oldToNewSub.setSize(localSize, -1);
-
- boolList sendElemIsUsed(localSize, false);
-
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = subMap_[domain];
- forAll(map, i)
- {
- if (recvFields[domain][i])
- {
- label index = map[i];
- if (subHasFlip_)
- {
- index = mag(index)-1;
- }
- sendElemIsUsed[index] = true;
- }
- }
- }
-
- label newI = 0;
- forAll(sendElemIsUsed, i)
- {
- if (sendElemIsUsed[i])
- {
- oldToNewSub[i] = newI++;
- }
- }
- }
-
-
- // Compact out all submap entries that are referring to unused elements
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = subMap_[domain];
-
- labelList newMap(map.size());
- label newI = 0;
-
- forAll(map, i)
- {
- if (recvFields[domain][i])
- {
- // So element is used on destination side
- label index = map[i];
- label sign = 1;
- if (subHasFlip_)
- {
- if (index < 0)
- {
- sign = -1;
- }
- index = mag(index)-1;
- }
- label newIndex = oldToNewSub[index];
- if (subHasFlip_)
- {
- newIndex = sign*(newIndex+1);
- }
- newMap[newI++] = newIndex;
- }
- }
- newMap.setSize(newI);
- subMap_[domain].transfer(newMap);
- }
- }
-
-
- // 2. remove from construct map - since end-result (element in elemIsUsed)
- // not used.
-
-
- oldToNewConstruct.setSize(elemIsUsed.size(), -1);
- constructSize_ = 0;
- forAll(elemIsUsed, i)
- {
- if (elemIsUsed[i])
- {
- oldToNewConstruct[i] = constructSize_++;
- }
- }
-
- for (const int domain : Pstream::allProcs(comm_))
- {
- const labelList& map = constructMap_[domain];
-
- labelList newMap(map.size());
- label newI = 0;
-
- forAll(map, i)
- {
- label destinationI = map[i];
- label sign = 1;
- if (constructHasFlip_)
- {
- if (destinationI < 0)
- {
- sign = -1;
- }
- destinationI = mag(destinationI)-1;
- }
-
- // Is element is used on destination side
- if (elemIsUsed[destinationI])
- {
- label newIndex = oldToNewConstruct[destinationI];
- if (constructHasFlip_)
- {
- newIndex = sign*(newIndex+1);
- }
- newMap[newI++] = newIndex;
- }
- }
- newMap.setSize(newI);
- constructMap_[domain].transfer(newMap);
- }
-}
-
-
// * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * //
void Foam::mapDistributeBase::operator=(const mapDistributeBase& rhs)
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.H b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.H
index 63490af5307..30908213a83 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.H
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.H
@@ -186,6 +186,34 @@ protected:
labelList& compactStart
);
+ //- Synchronize send/recv mask buffers as a 'copy' operation.
+ // Somewhat similar to Pstream::exchangeContainer
+ //
+ // The masks must be properly sized by the caller, which avoids
+ // a needless all-to-all for the sizes and the sizing is already
+ // given by the maps.
+ static void exchangeMasks
+ (
+ const UList<bitSet>& sendMasks,
+ UList<bitSet>& recvMasks,
+ const int tag,
+ const label comm
+ );
+
+ //- Bi-direction sync of send/recv buffers using bitwise '&='
+ //- combine operation.
+ //
+ // The masks must be properly sized by the caller, which avoids
+ // a needless all-to-all for the sizes and the sizing is already
+ // given by the maps.
+ static void unionCombineMasks
+ (
+ UList<bitSet>& sendMasks,
+ UList<bitSet>& recvMasks,
+ const int tag,
+ const label comm
+ );
+
template<class T, class CombineOp, class NegateOp>
static void flipAndCombine
@@ -225,6 +253,88 @@ private:
// Private Member Functions
+ //- Helper for compactData (private: filescope only!)
+ // Establishes the exact send/recv elements used after masking.
+ //
+ // \param allowedLocalElems Permissible local mapped elements
+ // (true/false). Can be longer/shorter than actual number
+ // of mapped elements.
+ // \param allowedRemoteElems Permissible remote mapped elements
+ // (true/false). Can be longer/shorter than actual number
+ // of mapped elements.
+ // \param[out] sendMasks Mask of local elements sent to procs.
+ // \param[out] recvMasks Mask of remote elements received
+ // from procs
+ // \param tag The message tag
+ void calcCompactDataRequirements
+ (
+ const bitSet& allowedLocalElems,
+ const bitSet& allowedRemoteElems,
+ List<bitSet>& sendMasks, // [out]
+ List<bitSet>& recvMasks, // [out]
+ const int tag
+ );
+
+ //- Helper for compactLocalData (private: filescope only!)
+ // Establishes the exact send/recv elements used after masking.
+ //
+ // \param allowedLocalElems Permissible local mapped elements
+ // (true/false). Can be longer/shorter than actual number
+ // of mapped elements.
+ // \param[out] sendMasks Mask of local elements sent to procs.
+ // \param[out] recvMasks Mask of remote elements received by proc.
+ // from procs
+ // \param tag The message tag
+ void calcCompactLocalDataRequirements
+ (
+ const bitSet& allowedLocalElems,
+ List<bitSet>& sendMasks, // [out]
+ List<bitSet>& recvMasks, // [out]
+ const int tag
+ );
+
+ //- Helper for compactRemoteData (private: filescope only!)
+ // Establishes the exact send/recv elements used after masking.
+ //
+ // \param allowedRemoteElems Permissible remote mapped elements
+ // (true/false). Can be longer/shorter than actual number
+ // of mapped elements.
+ // \param[out] sendMasks Mask of local elements sent to procs.
+ // \param[out] recvMasks Mask of remote elements received by proc.
+ // \param tag The message tag
+ void calcCompactRemoteDataRequirements
+ (
+ const bitSet& allowedRemoteElems,
+ List<bitSet>& sendMasks, // [out]
+ List<bitSet>& recvMasks, // [out]
+ const int tag
+ );
+
+ //- Implementation for compact{Local,Remote}Data (private).
+ // Also renumbers the subMap/constructMap if oldToNew maps
+ // are notNull().
+ //
+ // No communication
+ void compactData
+ (
+ const UList<bitSet>& sendMasks,
+ const UList<bitSet>& recvMasks,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize = -1
+ );
+
+ //- Wrapper for compactData (private) that supplies oldToNew
+ //- maps for renumbering if doRenumber is true.
+ // No communication
+ void compactDataImpl
+ (
+ const UList<bitSet>& sendMasks,
+ const UList<bitSet>& recvMasks,
+ const bool doRenumber
+ );
+
+
//- Helper for renumbering compacted map elements and updating the
//- supplied old-to-new mapping to account for the visit order of
//- the original elements
@@ -509,22 +619,163 @@ public:
// Compaction
- //- Compact all maps and layout.
- // Returns compaction maps for subMap and constructMap
- void compact
+ //- Compact send/receive maps based on selection of
+ //- originating local (send) elements.
+ // Determines and removes the correspondingly unneeded elements
+ // in the send/receive maps.
+ // Only compacts the maps, does not change the local layout.
+ //
+ // \param allowedLocalElems Permissible local mapped elements
+ // (true/false). Can be longer/shorter than actual number
+ // of mapped elements.
+ // \param tag The message tag
+ // \param doRenumber Apply oldToNew internally to renumber
+ // entries (uses renumberMap) and adjust the constructSize
+ //
+ // \note generally preferable to compact based on remote data
+ // (ie, the actual receiver)
+ void compactLocalData
+ (
+ const bitSet& allowedLocalElems,
+ const int tag = UPstream::msgType(),
+ const bool doRenumber = false
+ );
+
+ //- Compact send/receive maps based on selection of
+ //- remote (receive) elements.
+ // Determines and removes the correspondingly unneeded elements
+ // in the send/receive maps.
+ // Only compacts the maps, does not change the local layout.
+ //
+ // \param allowedRemoteElems Permissible remote mapped elements
+ // (true/false). Can be longer/shorter than actual number
+ // of mapped elements.
+ // \param tag The message tag
+ // \param doRenumber Apply oldToNew internally to renumber
+ // entries (uses renumberMap) and adjust the constructSize
+ void compactRemoteData
+ (
+ const bitSet& allowedRemoteElems,
+ const int tag = UPstream::msgType(),
+ const bool doRenumber = false
+ );
+
+
+ //- Compact send/receive maps based on selection of
+ //- originating local (send) elements.
+ //- Returns compaction mappings for subMap and constructMap.
+ //
+ // \param allowedLocalElems Permissible local mapped elements
+ // (true/false). Can be longer/shorter than actual number
+ // of mapped elements.
+ // \param[out] oldToNewSub Old-to-new mapping: subMap
+ // Disabled if labelList::null(),
+ // \param[out] oldToNewConstruct Old-to-new mapping: constructMap
+ // Disabled if labelList::null(),
+ // \param localSize The max index for subMap (-1: auto-detect)
+ // \param tag The message tag
+ //
+ // \note Applies oldToNew to renumber entries
+ // (uses renumberMap) and adjust constructSize
+ //
+ // \note generally preferable to compact based on remote data
+ // (ie, the actual receiver)
+ void compactLocalData
(
- const boolList& elemIsUsed,
+ const bitSet& allowedLocalElems,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize = -1,
const int tag = UPstream::msgType()
);
- //- Compact all maps and layout.
- // Returns compaction maps for subMap and constructMap
- void compact
+ //- Compact send/receive maps based on selection of
+ //- remote (receive) elements.
+ //- Returns compaction mappings for subMap and constructMap.
+ //
+ // \param allowedRemoteElems Permissible remote mapped elements
+ // (true/false). Can be longer/shorter than actual number
+ // of mapped elements.
+ // \param[out] oldToNewSub Old-to-new mapping: subMap
+ // Disabled if labelList::null(),
+ // \param[out] oldToNewConstruct Old-to-new mapping: constructMap
+ // Disabled if labelList::null(),
+ // \param localSize The max index for subMap (-1: auto-detect)
+ // \param tag The message tag
+ //
+ // \note Applies oldToNew to renumber entries
+ // (uses renumberMap) and adjust constructSize
+ void compactRemoteData
+ (
+ const bitSet& allowedRemoteElems,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize = -1,
+ const int tag = UPstream::msgType()
+ );
+
+
+ //- Compact send/receive maps based on selection of
+ //- originating local (send) and remote (receive) elements.
+ //
+ // The resulting compact numbering:
+ // - \c subMap (and \c oldToNewSub) :
+ // will follow the original ordering of \c localElements.
+ // - \c constructMap (and \c oldToNewConstruct) :
+ // will follow the original ordering of \c remoteElements.
+ // .
+ // \warning ill-defined behaviour if \c localElements
+ // or \c remoteElements contains duplicates.
+ void compactData
+ (
+ const labelUList& localElements,
+ const labelUList& remoteElements,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize = -1,
+ const int tag = UPstream::msgType()
+ );
+
+ //- Compact send/receive maps based on selection of
+ //- originating local (send) elements.
+ //
+ // The resulting compact numbering:
+ // - \c subMap (and \c oldToNewSub) :
+ // will follow the original ordering of \c localElements.
+ // - \c constructMap (and \c oldToNewConstruct) :
+ // numbered in simple ascending order.
+ // .
+ // \warning ill-defined behaviour if \c localElements
+ // contains duplicates.
+ //
+ // \note generally preferable to compact based on remote data
+ // (ie, the actual receiver)
+ void compactLocalData
(
- const boolList& elemIsUsed,
- const label localSize, // max index for subMap
+ const labelUList& localElements,
labelList& oldToNewSub,
labelList& oldToNewConstruct,
+ const label localSize = -1,
+ const int tag = UPstream::msgType()
+ );
+
+ //- Compact send/receive maps based on selection of
+ //- remote (receive) elements.
+ //
+ // The resulting compact numbering:
+ // - \c subMap (and \c oldToNewSub) :
+ // numbered in simple ascending order.
+ // - \c constructMap (and \c oldToNewConstruct) :
+ // will follow the original ordering of \c remoteElements.
+ // .
+ // \warning ill-defined behaviour if \c remoteElements
+ // contains duplicates.
+ void compactRemoteData
+ (
+ const labelUList& remoteElements,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize = -1,
const int tag = UPstream::msgType()
);
@@ -745,6 +996,25 @@ public:
{
NotImplemented;
}
+
+ //- OpenFOAM-v2112 and earlier naming for compactRemoteData()
+ //- using boolList.
+ void compact
+ (
+ const boolList& remoteElemUsed,
+ const int tag = UPstream::msgType()
+ );
+
+ //- OpenFOAM-v2112 and earlier naming for compactRemoteData().
+ //- using boolList.
+ void compact
+ (
+ const boolList& remoteElemUsed,
+ const label localSize,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const int tag = UPstream::msgType()
+ );
};
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBaseIO.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBaseIO.C
index eaeda12c764..f86d5f6a4ee 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBaseIO.C
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBaseIO.C
@@ -36,25 +36,39 @@ namespace Foam
// The maps (labelListList) are not human-modifiable but if we need to
// inspect them in ASCII, it is much more convenient if each sub-list
// is flattened on a single line.
-static void writeMaps(Ostream& os, const word& key, const labelListList& maps)
+static Ostream& printMaps(Ostream& os, const labelListList& maps)
{
- if (os.format() == IOstream::BINARY)
+ if (os.format() == IOstream::BINARY || maps.empty())
{
- os.writeEntry(key, maps);
+ os << maps;
}
else
{
- os << indent << key << nl
- << maps.size() << nl
+ os << nl << maps.size() << nl
<< token::BEGIN_LIST << nl;
- // Single-line output
+ // Compact single-line output for each labelList
for (const labelList& map : maps)
{
map.writeList(os) << nl;
}
+ os << token::END_LIST;
+ }
+
+ return os;
+}
- os << token::END_LIST << token::END_STATEMENT << nl;
+
+static void writeMaps(Ostream& os, const word& key, const labelListList& maps)
+{
+ if (os.format() == IOstream::BINARY || maps.empty())
+ {
+ os.writeEntry(key, maps);
+ }
+ else
+ {
+ os << indent << key;
+ printMaps(os, maps) << token::END_STATEMENT << nl;
}
}
@@ -129,7 +143,8 @@ Foam::Istream& Foam::operator>>(Istream& is, mapDistributeBase& map)
{
is.fatalCheck(FUNCTION_NAME);
- is >> map.constructSize_ >> map.subMap_ >> map.constructMap_
+ is >> map.constructSize_
+ >> map.subMap_ >> map.constructMap_
>> map.subHasFlip_ >> map.constructHasFlip_
>> map.comm_;
@@ -139,11 +154,14 @@ Foam::Istream& Foam::operator>>(Istream& is, mapDistributeBase& map)
Foam::Ostream& Foam::operator<<(Ostream& os, const mapDistributeBase& map)
{
- os << map.constructSize_ << token::NL
- << map.subMap_ << token::NL
- << map.constructMap_ << token::NL
- << map.subHasFlip_ << token::SPACE << map.constructHasFlip_
- << token::SPACE << map.comm_ << token::NL;
+ os << map.constructSize_ << token::NL;
+
+ printMaps(os, map.subMap_) << token::NL;
+ printMaps(os, map.constructMap_) << token::NL;
+
+ os << map.subHasFlip_ << token::SPACE
+ << map.constructHasFlip_ << token::SPACE
+ << map.comm_ << token::NL;
return os;
}
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBaseSubset.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBaseSubset.C
index 67938842900..9eedb3ee3d4 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBaseSubset.C
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBaseSubset.C
@@ -29,6 +29,252 @@ License
#include "bitSet.H"
#include "ListOps.H"
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Setup array of element masks to match maps sizes
+//
+// \param[out] masks Sized for each position in the maps
+// \param maps The element maps
+static void blankElementMasks(List<bitSet>& masks, const labelListList& maps)
+{
+ // If base container not already sized
+ if (masks.empty())
+ {
+ masks.resize(maps.size());
+ }
+
+ forAll(masks, proci)
+ {
+ masks[proci].reset(); // zero all bits
+ masks[proci].resize(maps[proci].size());
+ }
+}
+
+
+// Calculate the element mask correspondig to allowedElems in the maps
+//
+// \param allowedElems Permissible mapped elements (true/false)
+// \param[out] masks True/false for each position within the maps
+// \param maps The element maps
+// \param hasFlip Map has flip indexing
+//
+// \return the max index used.
+static label calcElementMasks
+(
+ const bitSet& allowedElems,
+ List<bitSet>& masks, // [out] - often presized before calling
+ const labelListList& maps,
+ const bool hasFlip
+)
+{
+ // Index after flipping
+ const auto unflippedIndex =
+ (
+ hasFlip
+ ? [](label idx) -> label { return mag(idx)-1; }
+ : [](label idx) -> label { return idx; }
+ );
+
+
+ // If not already sized
+ if (masks.empty())
+ {
+ masks.resize(maps.size());
+ }
+
+ label maxIndex = -1;
+
+ forAll(masks, proci)
+ {
+ bitSet& mask = masks[proci];
+ const labelList& map = maps[proci];
+
+ mask.reset(); // zero all bits
+ mask.resize(map.size());
+
+ forAll(map, i)
+ {
+ // Element is used (or not)
+ const label index = unflippedIndex(map[i]);
+
+ if (allowedElems.test(index))
+ {
+ mask.set(i);
+ maxIndex = max(maxIndex, index);
+ }
+ }
+ }
+
+ return maxIndex;
+}
+
+} // End namespace Foam
+
+
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
+
+void Foam::mapDistributeBase::exchangeMasks
+(
+ const UList<bitSet>& sendMasks,
+ UList<bitSet>& recvMasks,
+ const int tag,
+ const label comm
+)
+{
+ // Require properly sized mask buffers.
+ // The information *is* known from the maps, so always use that
+ // instead having a needless all-to-all for the sizes.
+
+ if (sendMasks.size() != recvMasks.size())
+ {
+ FatalErrorInFunction
+ << "Mismatched mask sizes: "
+ << sendMasks.size() << " != "
+ << recvMasks.size() << nl
+ << Foam::abort(FatalError);
+ }
+
+ const label myRank = UPstream::myProcNo(comm);
+
+ if (UPstream::parRun())
+ {
+ #ifdef FULLDEBUG
+ if (sendMasks.size() > UPstream::nProcs(comm))
+ {
+ FatalErrorInFunction
+ << "Mask sizes (" << sendMasks.size()
+ << ") are larger than number of procs:"
+ << UPstream::nProcs(comm) << nl
+ << Foam::abort(FatalError);
+ }
+ #endif
+
+ const label startOfRequests = UPstream::nRequests();
+
+ forAll(recvMasks, proci)
+ {
+ if (proci != myRank && recvMasks[proci].size())
+ {
+ IPstream::read
+ (
+ UPstream::commsTypes::nonBlocking,
+ proci,
+ recvMasks[proci].data_bytes(),
+ recvMasks[proci].size_bytes(),
+ tag,
+ comm
+ );
+ }
+ }
+
+ forAll(sendMasks, proci)
+ {
+ if (proci != myRank && sendMasks[proci].size())
+ {
+ OPstream::write
+ (
+ UPstream::commsTypes::nonBlocking,
+ proci,
+ sendMasks[proci].cdata_bytes(),
+ sendMasks[proci].size_bytes(),
+ tag,
+ comm
+ );
+ }
+ }
+
+ // Wait for all to finish
+ Pstream::waitRequests(startOfRequests);
+ }
+
+ // Receiving myself is just a copy
+ recvMasks[myRank] = sendMasks[myRank];
+}
+
+
+void Foam::mapDistributeBase::unionCombineMasks
+(
+ UList<bitSet>& sendMasks,
+ UList<bitSet>& recvMasks,
+ const int tag,
+ const label comm
+)
+{
+ // Require properly sized mask buffers.
+ // The information *is* known from the maps, so always use that
+ // instead having a needless all-to-all for the sizes.
+
+ if (sendMasks.size() != recvMasks.size())
+ {
+ FatalErrorInFunction
+ << "Mismatched mask sizes: "
+ << sendMasks.size() << " != "
+ << recvMasks.size() << nl
+ << Foam::abort(FatalError);
+ }
+
+ if (Pstream::parRun())
+ {
+ // Scratch buffers for union operations
+ List<bitSet> scratch(recvMasks.size());
+
+ // Size for receives
+ forAll(scratch, proci)
+ {
+ scratch[proci].resize(recvMasks[proci].size());
+ }
+
+ // Exchange: from sendMasks -> scratch (intermediate receive)
+ exchangeMasks(sendMasks, scratch, tag, comm);
+
+ // Update recvMasks (as union)
+ forAll(recvMasks, proci)
+ {
+ recvMasks[proci] &= scratch[proci];
+ }
+
+ // Size for sends
+ forAll(scratch, proci)
+ {
+ scratch[proci].resize(sendMasks[proci].size());
+ }
+
+ // Exchange: from recvMasks -> scratch (intermediate send)
+ exchangeMasks(recvMasks, scratch, tag, comm);
+
+ // Final synchronization
+ forAll(sendMasks, proci)
+ {
+ sendMasks[proci] &= scratch[proci];
+ }
+ }
+ else
+ {
+ // Non-parallel: 'synchronize' myself
+ const label myRank = Pstream::myProcNo(comm);
+
+ recvMasks[myRank] &= sendMasks[myRank];
+ sendMasks[myRank] = recvMasks[myRank];
+ }
+
+ // Done with parallel exchanges so can shrink the masks to
+ // the min-size actually needed.
+
+ for (auto& mask : sendMasks)
+ {
+ mask.resize_last();
+ }
+
+ for (auto& mask : recvMasks)
+ {
+ mask.resize_last();
+ }
+}
+
+
// * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * * //
Foam::label Foam::mapDistributeBase::renumberMap
@@ -128,4 +374,568 @@ void Foam::mapDistributeBase::renumberVisitOrder
}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+void Foam::mapDistributeBase::calcCompactDataRequirements
+(
+ const bitSet& allowedLocalElems,
+ const bitSet& allowedRemoteElems,
+ List<bitSet>& sendMasks, // [out]
+ List<bitSet>& recvMasks, // [out]
+ const int tag
+)
+{
+ sendMasks.resize_nocopy(UPstream::nProcs(comm_));
+ recvMasks.resize_nocopy(UPstream::nProcs(comm_));
+
+ // Determine local elements sent to which procs
+ calcElementMasks
+ (
+ allowedLocalElems,
+ sendMasks,
+ subMap_,
+ subHasFlip_
+ );
+
+ // Determine remote elements received from which procs
+ calcElementMasks
+ (
+ allowedRemoteElems,
+ recvMasks,
+ constructMap_,
+ constructHasFlip_
+ );
+
+ // Synchronize - combine as '&' union
+ unionCombineMasks(sendMasks, recvMasks, tag, comm_);
+}
+
+
+void Foam::mapDistributeBase::calcCompactLocalDataRequirements
+(
+ const bitSet& allowedLocalElems,
+ List<bitSet>& sendMasks, // [out]
+ List<bitSet>& recvMasks, // [out]
+ const int tag
+)
+{
+ sendMasks.resize_nocopy(UPstream::nProcs(comm_));
+ recvMasks.resize_nocopy(UPstream::nProcs(comm_));
+
+ // Determine local elements sent to which procs
+ calcElementMasks
+ (
+ allowedLocalElems,
+ sendMasks,
+ subMap_,
+ subHasFlip_
+ );
+
+ blankElementMasks(recvMasks, constructMap_);
+
+ // Exchange: from sendMasks -> recvMasks
+ exchangeMasks(sendMasks, recvMasks, tag, comm_);
+}
+
+
+void Foam::mapDistributeBase::calcCompactRemoteDataRequirements
+(
+ const bitSet& allowedRemoteElems,
+ List<bitSet>& sendMasks, // [out]
+ List<bitSet>& recvMasks, // [out]
+ const int tag
+)
+{
+ sendMasks.resize_nocopy(UPstream::nProcs(comm_));
+ recvMasks.resize_nocopy(UPstream::nProcs(comm_));
+
+ // Determine remote elements received from which procs
+ calcElementMasks
+ (
+ allowedRemoteElems,
+ recvMasks,
+ constructMap_,
+ constructHasFlip_
+ );
+
+ blankElementMasks(sendMasks, subMap_);
+
+ // Exchange: from recvMasks -> sendMasks
+ exchangeMasks(recvMasks, sendMasks, tag, comm_);
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+void Foam::mapDistributeBase::compactData
+(
+ const UList<bitSet>& sendMasks,
+ const UList<bitSet>& recvMasks,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize // (known) max sizing for subMap
+)
+{
+ // Linear address (subMap) after any flipping
+ const auto unflippedSendIndex =
+ (
+ subHasFlip_
+ ? [](label idx) -> label { return mag(idx)-1; }
+ : [](label idx) -> label { return idx; }
+ );
+
+ // Linear address (constructMap) after any flipping
+ const auto unflippedRecvIndex =
+ (
+ constructHasFlip_
+ ? [](label idx) -> label { return mag(idx)-1; }
+ : [](label idx) -> label { return idx; }
+ );
+
+
+ // Compact renumbering enabled if oldToNew maps are notNull
+
+ bitSet indexUsed;
+
+ // The subMap old-to-new mapping
+ if (notNull(oldToNewSub))
+ {
+ label subMapSize(localSize);
+ if (subMapSize < 0)
+ {
+ subMapSize = getMappedSize(subMap_, subHasFlip_);
+ }
+
+ oldToNewSub.resize_nocopy(subMapSize);
+ oldToNewSub = -1;
+
+ indexUsed.reset(); // zero all bits
+ indexUsed.resize(subMapSize);
+
+ forAll(sendMasks, proci)
+ {
+ const bitSet& mask = sendMasks[proci];
+ const auto& map = subMap_[proci];
+
+ for (const label i : mask)
+ {
+ const label index = unflippedSendIndex(map[i]);
+
+ indexUsed.set(index);
+ }
+ }
+
+ label nCompact = 0;
+ for (const label i : indexUsed)
+ {
+ oldToNewSub[i] = nCompact++;
+ }
+ }
+
+
+ // The constructMap old-to-new mapping
+ if (notNull(oldToNewConstruct))
+ {
+ oldToNewConstruct.resize_nocopy(constructSize_);
+ oldToNewConstruct = -1;
+
+ indexUsed.reset(); // zero all bits
+ indexUsed.resize(constructSize_);
+
+ forAll(recvMasks, proci)
+ {
+ const bitSet& mask = recvMasks[proci];
+ const auto& map = constructMap_[proci];
+
+ for (const label i : mask)
+ {
+ const label index = unflippedRecvIndex(map[i]);
+
+ indexUsed.set(index);
+ }
+ }
+
+ label nCompact = 0;
+ for (const label i : indexUsed)
+ {
+ oldToNewConstruct[i] = nCompact++;
+ }
+ }
+
+
+ // Compact out subMap entries referring to unused elements
+ forAll(sendMasks, proci)
+ {
+ const bitSet& mask = sendMasks[proci];
+ labelList& map = subMap_[proci];
+
+ label nCompact = 0;
+
+ for (const label i : mask)
+ {
+ // const label index = unflippedSendIndex(map[i]);
+ // maxLocalIndex = max(maxLocalIndex, index);
+
+ map[nCompact++] = map[i];
+ }
+
+ map.resize(nCompact);
+ }
+
+
+ // Compact out constructMap entries referring to unused elements
+
+ label maxRemoteIndex = -1;
+
+ forAll(recvMasks, proci)
+ {
+ const bitSet& mask = recvMasks[proci];
+ labelList& map = constructMap_[proci];
+
+ label nCompact = 0;
+
+ for (const label i : mask)
+ {
+ const label index = unflippedRecvIndex(map[i]);
+ maxRemoteIndex = max(maxRemoteIndex, index);
+
+ map[nCompact++] = map[i];
+ }
+
+ map.resize(nCompact);
+ }
+
+ constructSize_ = maxRemoteIndex+1;
+
+
+ // Do compact renumbering...
+
+ if (notNull(oldToNewSub))
+ {
+ renumberMap(subMap_, oldToNewSub, subHasFlip_);
+ }
+
+ if (notNull(oldToNewConstruct))
+ {
+ constructSize_ =
+ renumberMap(constructMap_, oldToNewConstruct, constructHasFlip_);
+ }
+
+ // Clear the schedule (note:not necessary if nothing changed)
+ schedulePtr_.reset(nullptr);
+}
+
+
+void Foam::mapDistributeBase::compactData
+(
+ const labelUList& localElements,
+ const labelUList& remoteElements,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize,
+ const int tag
+)
+{
+ List<bitSet> sendMasks;
+ List<bitSet> recvMasks;
+
+ calcCompactDataRequirements
+ (
+ bitSet(localElements),
+ bitSet(remoteElements),
+ sendMasks,
+ recvMasks,
+ tag
+ );
+
+ // Perform compaction and renumbering
+ compactData
+ (
+ sendMasks,
+ recvMasks,
+ oldToNewSub,
+ oldToNewConstruct,
+ localSize
+ );
+
+ // Renumber according to visit order
+ renumberVisitOrder
+ (
+ localElements,
+ oldToNewSub,
+ subMap_,
+ subHasFlip_
+ );
+
+ // Renumber according to visit order
+ renumberVisitOrder
+ (
+ remoteElements,
+ oldToNewConstruct,
+ constructMap_,
+ constructHasFlip_
+ );
+}
+
+
+void Foam::mapDistributeBase::compactLocalData
+(
+ const labelUList& localElements,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize,
+ const int tag
+)
+{
+ List<bitSet> sendMasks;
+ List<bitSet> recvMasks;
+
+ calcCompactLocalDataRequirements
+ (
+ // Retain items required on the local side
+ bitSet(localElements),
+ sendMasks,
+ recvMasks,
+ tag
+ );
+
+ // Perform compaction and renumbering
+ compactData
+ (
+ sendMasks,
+ recvMasks,
+ oldToNewSub,
+ oldToNewConstruct,
+ localSize
+ );
+
+ // Renumber according to visit order
+ renumberVisitOrder
+ (
+ localElements,
+ oldToNewSub,
+ subMap_,
+ subHasFlip_
+ );
+}
+
+
+void Foam::mapDistributeBase::compactRemoteData
+(
+ const labelUList& remoteElements,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize,
+ const int tag
+)
+{
+ List<bitSet> sendMasks;
+ List<bitSet> recvMasks;
+
+ calcCompactRemoteDataRequirements
+ (
+ // Retain items required on the remote side
+ bitSet(remoteElements),
+ sendMasks,
+ recvMasks,
+ tag
+ );
+
+ // Perform compaction and renumbering
+ compactData
+ (
+ sendMasks,
+ recvMasks,
+ oldToNewSub,
+ oldToNewConstruct,
+ localSize
+ );
+
+ // Renumber according to visit order
+ renumberVisitOrder
+ (
+ remoteElements,
+ oldToNewConstruct,
+ constructMap_,
+ constructHasFlip_
+ );
+}
+
+
+void Foam::mapDistributeBase::compactDataImpl
+(
+ const UList<bitSet>& sendMasks,
+ const UList<bitSet>& recvMasks,
+ const bool doRenumber
+)
+{
+ if (doRenumber)
+ {
+ labelList oldToNewSub;
+ labelList oldToNewConstruct;
+
+ compactData
+ (
+ sendMasks,
+ recvMasks,
+ oldToNewSub,
+ oldToNewConstruct,
+ -1 // localSize: automatic
+ );
+ }
+ else
+ {
+ // Call with placeholder values
+ compactData
+ (
+ sendMasks,
+ recvMasks,
+ const_cast<labelList&>(labelList::null()), // disabled
+ const_cast<labelList&>(labelList::null()), // disabled
+ -1 // localSize: automatic
+ );
+ }
+}
+
+
+void Foam::mapDistributeBase::compactLocalData
+(
+ const bitSet& allowedLocalElems,
+ const int tag,
+ const bool doRenumber
+)
+{
+ List<bitSet> sendMasks;
+ List<bitSet> recvMasks;
+
+ calcCompactLocalDataRequirements
+ (
+ allowedLocalElems,
+ sendMasks,
+ recvMasks,
+ tag
+ );
+
+ compactDataImpl(sendMasks, recvMasks, doRenumber);
+}
+
+
+void Foam::mapDistributeBase::compactRemoteData
+(
+ const bitSet& allowedRemoteElems,
+ const int tag,
+ const bool doRenumber
+)
+{
+ List<bitSet> sendMasks;
+ List<bitSet> recvMasks;
+
+ calcCompactRemoteDataRequirements
+ (
+ allowedRemoteElems,
+ sendMasks,
+ recvMasks,
+ tag
+ );
+
+ compactDataImpl(sendMasks, recvMasks, doRenumber);
+}
+
+
+void Foam::mapDistributeBase::compactLocalData
+(
+ const bitSet& allowedLocalElems,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize,
+ const int tag
+)
+{
+ List<bitSet> sendMasks;
+ List<bitSet> recvMasks;
+
+ calcCompactLocalDataRequirements
+ (
+ allowedLocalElems,
+ sendMasks,
+ recvMasks,
+ tag
+ );
+
+ compactData
+ (
+ sendMasks,
+ recvMasks,
+ oldToNewSub,
+ oldToNewConstruct,
+ localSize
+ );
+}
+
+
+void Foam::mapDistributeBase::compactRemoteData
+(
+ const bitSet& allowedRemoteElems,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const label localSize,
+ const int tag
+)
+{
+ List<bitSet> sendMasks;
+ List<bitSet> recvMasks;
+
+ calcCompactRemoteDataRequirements
+ (
+ allowedRemoteElems,
+ sendMasks,
+ recvMasks,
+ tag
+ );
+
+ compactData
+ (
+ sendMasks,
+ recvMasks,
+ oldToNewSub,
+ oldToNewConstruct,
+ localSize
+ );
+}
+
+
+// * * * * * * * * * * * * * * * Housekeeping * * * * * * * * * * * * * * * //
+
+void Foam::mapDistributeBase::compact
+(
+ const boolList& remoteElemUsed,
+ const int tag
+)
+{
+ // Forward to bitSet version
+ compactRemoteData(bitSet(remoteElemUsed), tag);
+}
+
+
+void Foam::mapDistributeBase::compact
+(
+ const boolList& remoteElemUsed,
+ const label localSize,
+ labelList& oldToNewSub,
+ labelList& oldToNewConstruct,
+ const int tag
+)
+{
+ // Forward to bitSet version
+ compactRemoteData
+ (
+ bitSet(remoteElemUsed),
+ oldToNewSub,
+ oldToNewConstruct,
+ localSize,
+ tag
+ );
+}
+
+
// ************************************************************************* //
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMeshIO.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMeshIO.C
index 1fb1b90e874..50e1663f03b 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMeshIO.C
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMeshIO.C
@@ -150,9 +150,11 @@ Foam::Istream& Foam::operator>>(Istream& is, mapDistributePolyMesh& map)
is >> map.nOldPoints_
>> map.nOldFaces_
>> map.nOldCells_
+
>> map.oldPatchSizes_
>> map.oldPatchStarts_
>> map.oldPatchNMeshPoints_
+
>> map.pointMap_
>> map.faceMap_
>> map.cellMap_
@@ -171,6 +173,7 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const mapDistributePolyMesh& map)
<< map.oldPatchSizes_ << token::NL
<< map.oldPatchStarts_ << token::NL
<< map.oldPatchNMeshPoints_ << token::NL
+
<< map.pointMap_ << token::NL
<< map.faceMap_ << token::NL
<< map.cellMap_ << token::NL
--
GitLab