Commit f44dbbc2 authored by Mattijs Janssens's avatar Mattijs Janssens Committed by Andrew Heather
Browse files

ENH: Support AMI for multi-world operation. Fixes #2099

Multi-world operation now supports AMI:

    // What to sample:
    sampleMode      nearestPatchFaceAMI;
parent 9a3d27e3
......@@ -67,6 +67,7 @@ void Foam::mappedPatchFieldBase<Type>::storeField
for (label domain = 0; domain < nProcs; domain++)
{
const labelList& map = procToMap[domain];
if (map.size())
{
const Field<T> subFld(fld, map);
......@@ -100,7 +101,7 @@ void Foam::mappedPatchFieldBase<Type>::storeField
template<class Type>
template<class T>
void Foam::mappedPatchFieldBase<Type>::retrieveField
bool Foam::mappedPatchFieldBase<Type>::retrieveField
(
const bool allowUnset,
const objectRegistry& obr,
......@@ -112,9 +113,10 @@ void Foam::mappedPatchFieldBase<Type>::retrieveField
) const
{
// Store my data onto database
//const label myRank = Pstream::myProcNo(0); // comm_
const label nProcs = Pstream::nProcs(0); // comm_
bool ok = true;
for (label domain = 0; domain < nProcs; domain++)
{
const labelList& map = procToMap[domain];
......@@ -128,39 +130,68 @@ void Foam::mappedPatchFieldBase<Type>::retrieveField
/ patch
);
//const IOField<T>& subFld = subObr.lookupObject<IOField<T>>
//(
// fieldName
//);
const IOField<T>* subFldPtr = subObr.getObjectPtr<IOField<T>>
(
fieldName
);
if (subFldPtr)
{
UIndirectList<T>(fld, map) = *subFldPtr;
if (fvPatchField<Type>::debug)
if (subFldPtr->size() != map.size())
{
Pout<< "*** RETRIEVED :"
<< " field:" << fieldName
<< " values:" << flatOutput(fld)
<< " from:" << subObr.objectPath() << endl;
// This is the dummy value inserted at start-up since the
// map is always non-zero size (checked above)
//Pout<< "*** RETRIEVED DUMMY :"
// << " field:" << fieldName
// << " subFldPtr:" << subFldPtr->size()
// << " map:" << map.size() << endl;
ok = false;
}
else
{
UIndirectList<T>(fld, map) = *subFldPtr;
if (fvPatchField<Type>::debug)
{
Pout<< "*** RETRIEVED :"
<< " field:" << fieldName
<< " values:" << flatOutput(fld)
<< " from:" << subObr.objectPath() << endl;
}
}
}
else if (allowUnset)
{
WarningInFunction << "Not found"
<< " field:" << fieldName
<< " in:" << subObr.objectPath() << endl;
if (fvPatchField<Type>::debug)
{
WarningInFunction << "Not found"
<< " field:" << fieldName
<< " in:" << subObr.objectPath() << endl;
}
// Store dummy value so the database has something on it.
// Note that size 0 should never occur naturally so we can
// detect it if necessary.
const Field<T> dummyFld(0);
mappedPatchBase::storeField
(
const_cast<objectRegistry&>(subObr),
fieldName,
dummyFld
);
ok = false;
}
else
{
// Not found. Make it fail
(void)subObr.lookupObject<IOField<T>>(fieldName);
ok = false;
}
}
}
return ok;
}
......@@ -171,18 +202,17 @@ void Foam::mappedPatchFieldBase<Type>::initRetrieveField
const objectRegistry& obr,
const word& region,
const word& patch,
const mapDistribute& map,
const labelListList& map,
const word& fieldName,
const Field<T>& fld
) const
{
// Store my data onto database
//const label myRank = Pstream::myProcNo(0); // comm_
const label nProcs = Pstream::nProcs(0); // comm_
for (label domain = 0; domain < nProcs; domain++)
{
const labelList& constructMap = map.constructMap()[domain];
const labelList& constructMap = map[domain];
if (constructMap.size())
{
const objectRegistry& subObr = mappedPatchBase::subRegistry
......@@ -216,6 +246,71 @@ void Foam::mappedPatchFieldBase<Type>::initRetrieveField
}
template<class Type>
template<class T>
bool Foam::mappedPatchFieldBase<Type>::storeAndRetrieveField
(
const word& fieldName,
const labelListList& subMap,
const label constructSize,
const labelListList& constructMap,
const labelListList& address,
const scalarListList& weights,
Field<T>& fld
) const
{
storeField
(
patchField_.internalField().time(),
patchField_.patch().boundaryMesh().mesh().name(),
patchField_.patch().name(),
subMap,
fieldName,
fld
);
Field<T> work(constructSize);
const bool ok = retrieveField
(
true, // allow unset
patchField_.internalField().time(),
mapper_.sampleRegion(),
mapper_.samplePatch(),
constructMap,
fieldName,
work
);
if (ok)
{
// Do interpolation
fld.setSize(address.size());
fld = Zero;
const plusEqOp<T> cop;
const multiplyWeightedOp<T, plusEqOp<T>> mop(cop);
forAll(address, facei)
{
const labelList& slots = address[facei];
const scalarList& w = weights[facei];
forAll(slots, i)
{
mop(fld[facei], facei, work[slots[i]], w[i]);
}
}
}
else
{
// Leave fld intact
}
return ok;
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
template<class Type>
......@@ -263,16 +358,24 @@ Foam::mappedPatchFieldBase<Type>::mappedPatchFieldBase
if
(
mapper_.sampleDatabase()
&& mapper_.mode() != mappedPatchBase::NEARESTPATCHFACE
&& (
mapper_.mode() != mappedPatchBase::NEARESTPATCHFACE
&& mapper_.mode() != mappedPatchBase::NEARESTPATCHFACEAMI
)
)
{
FatalErrorInFunction
<< "Mapping using the database only supported for "
<< "sampleMode "
<< "sampleModes "
<< mappedPatchBase::sampleModeNames_
[
mappedPatchBase::NEARESTPATCHFACE
]
<< " and "
<< mappedPatchBase::sampleModeNames_
[
mappedPatchBase::NEARESTPATCHFACEAMI
]
<< exit(FatalError);
}
......@@ -297,22 +400,29 @@ Foam::mappedPatchFieldBase<Type>::mappedPatchFieldBase
:
mappedPatchFieldBase<Type>::mappedPatchFieldBase(mapper, patchField, dict)
{
if
(
mapper_.mode() == mappedPatchBase::NEARESTPATCHFACE
&& mapper_.sampleDatabase()
)
if (mapper_.sampleDatabase())
{
// Store my data on receive buffers so we have some initial data
initRetrieveField
(
patchField_.internalField().time(),
patchField_.patch().boundaryMesh().mesh().name(),
patchField_.patch().name(),
mapper_.map(),
patchField_.internalField().name(),
patchField_
);
if (mapper_.mode() == mappedPatchBase::NEARESTPATCHFACE)
{
// Store my data on receive buffers so we have some initial data
initRetrieveField
(
patchField_.internalField().time(),
//patchField_.patch().boundaryMesh().mesh().name(),
mapper_.sampleRegion(),
//patchField_.patch().name(),
mapper_.samplePatch(),
mapper_.map().constructMap(),
patchField_.internalField().name(),
patchField_
);
}
else if (mapper_.mode() == mappedPatchBase::NEARESTPATCHFACEAMI)
{
// Depend on fall-back (sorting dummy field) in retrieveField
// since it would be too hard to determine the field that gives
// the wanted result after interpolation
}
}
}
......@@ -410,37 +520,80 @@ template<class T>
void Foam::mappedPatchFieldBase<Type>::distribute
(
const word& fieldName,
Field<T>& newValues
Field<T>& fld
) const
{
if (mapper_.sampleDatabase())
{
// Store my data on send buffers
storeField
(
patchField_.internalField().time(),
patchField_.patch().boundaryMesh().mesh().name(),
patchField_.patch().name(),
mapper_.map().subMap(),
fieldName,
newValues
);
// Construct my data from receive buffers
newValues.setSize(mapper_.map().constructSize());
retrieveField
(
true, // allow unset
patchField_.internalField().time(),
mapper_.sampleRegion(),
mapper_.samplePatch(),
mapper_.map().constructMap(),
fieldName,
newValues
);
if (mapper_.mode() != mappedPatchBase::NEARESTPATCHFACEAMI)
{
// Store my data on send buffers
storeField
(
patchField_.internalField().time(),
patchField_.patch().boundaryMesh().mesh().name(),
patchField_.patch().name(),
mapper_.map().subMap(),
fieldName,
fld
);
// Construct my data from receive buffers
fld.setSize(mapper_.map().constructSize());
retrieveField
(
true, // allow unset
patchField_.internalField().time(),
mapper_.sampleRegion(),
mapper_.samplePatch(),
mapper_.map().constructMap(),
fieldName,
fld
);
}
else
{
const AMIPatchToPatchInterpolation& AMI = mapper_.AMI();
// The AMI does an interpolateToSource/ToTarget. This is a
// mapDistribute (so using subMap/constructMap) and then a
// weighted sum. We'll store the sent data as before and
// do the weighted summation after the retrieveField
if (mapper_.masterWorld())
{
// See AMIInterpolation::interpolateToSource. Use tgtMap,
// srcAddress, srcWeights
storeAndRetrieveField
(
fieldName,
AMI.srcMap().subMap(),
AMI.tgtMap().constructSize(),
AMI.tgtMap().constructMap(),
AMI.srcAddress(),
AMI.srcWeights(),
fld
);
}
else
{
// See AMIInterpolation::interpolateToTarget.
// Use srcMap, tgtAddress, tgtWeights
storeAndRetrieveField
(
fieldName,
AMI.tgtMap().subMap(),
AMI.srcMap().constructSize(),
AMI.srcMap().constructMap(),
AMI.tgtAddress(),
AMI.tgtWeights(),
fld
);
}
}
}
else
{
mapper_.distribute(newValues);
mapper_.distribute(fld);
}
}
......@@ -825,15 +978,18 @@ void Foam::mappedPatchFieldBase<Type>::initRetrieveField
{
// Store my data on receive buffers (reverse of storeField;
// i.e. retrieveField will obtain patchField)
initRetrieveField
(
patchField_.internalField().time(),
mapper_.sampleRegion(),
mapper_.samplePatch(),
mapper_.map(),
fieldName,
fld
);
if (mapper_.mode() == mappedPatchBase::NEARESTPATCHFACE)
{
initRetrieveField
(
patchField_.internalField().time(),
mapper_.sampleRegion(),
mapper_.samplePatch(),
mapper_.map().constructMap(),
fieldName,
fld
);
}
}
}
......
......@@ -6,6 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2013-2016 OpenFOAM Foundation
Copyright (C) 2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
......@@ -116,11 +117,26 @@ protected:
const objectRegistry& obr,
const word& region,
const word& patch,
const mapDistribute& map,
const labelListList& map,
const word& fieldName,
const Field<T>& fld
) const;
//- Helper : storeField and retrieveField and interpolate. Leaves fld
// unchanged (and returns false) if new values cannot be retrieved.
// Returns true otherwise.
template<class T>
bool storeAndRetrieveField
(
const word& fieldName,
const labelListList& subMap,
const label constructSize,
const labelListList& constructMap,
const labelListList& address,
const scalarListList& weights,
Field<T>& fld
) const;
public:
......@@ -244,9 +260,10 @@ public:
const Field<T>& fld
) const;
//- Construct field from registered elements
//- Construct field from registered elements. Return true if
// successful
template<class T>
void retrieveField
bool retrieveField
(
const bool allowUnset,
const objectRegistry& obr,
......
......@@ -1105,14 +1105,14 @@ void Foam::mappedPatchBase::calcAMI() const
}
// Check if running locally
if (sampleWorld_.empty())
if (sampleWorld_.empty() || sameWorld())
{
const polyPatch& nbr = samplePolyPatch();
// Transform neighbour patch to local system
pointField nbrPoints(samplePoints(nbr.localPoints()));
const pointField nbrPoints(samplePoints(nbr.localPoints()));
primitivePatch nbrPatch0
const primitivePatch nbrPatch0
(
SubList<face>
(
......@@ -1135,43 +1135,56 @@ void Foam::mappedPatchBase::calcAMI() const
meshTools::writeOBJ(osO, patch_.localFaces(), patch_.localPoints());
}
// Construct/apply AMI interpolation to determine addressing and weights
// Construct/apply AMI interpolation to determine addressing and
// weights. Make sure to use optional inter-world communicator.
const label oldWorldComm = Pstream::worldComm;
Pstream::worldComm = comm_;
const label oldComm(Pstream::warnComm);
Pstream::warnComm = UPstream::worldComm;
AMIPtr_->calculate(patch_, nbrPatch0, surfPtr());
Pstream::warnComm = oldComm;
Pstream::worldComm = oldWorldComm;
}
else
{
faceList nbrFaces;
pointField nbrPoints;
if (sampleWorld() == UPstream::myWorld())
{
const polyPatch& nbr = samplePolyPatch();
nbrFaces = nbr.localFaces();
nbrPoints = samplePoints(nbr.localPoints());
}
else
{
// Leave empty
}
primitivePatch nbrPatch0
faceList dummyFaces;
pointField dummyPoints;
const primitivePatch dummyPatch
(
SubList<face>
(
nbrFaces,
nbrFaces.size()
dummyFaces
),
nbrPoints
dummyPoints
);
// Change to use ALL processors communicator
// Change to use inter-world communicator
const label oldWorldComm = Pstream::worldComm;
Pstream::worldComm = comm_;
const label oldComm(Pstream::warnComm);
Pstream::warnComm = UPstream::worldComm;
// Construct/apply AMI interpolation to determine addressing and weights
AMIPtr_->calculate(patch_, nbrPatch0, surfPtr());
if (masterWorld())
{
// Construct/apply AMI interpolation to determine addressing
// and weights. Have patch_ for src faces, 0 faces for the
// target side
AMIPtr_->calculate(patch_, dummyPatch, surfPtr());
}
else
{
// Construct/apply AMI interpolation to determine addressing
// and weights. Have 0 faces for src side, patch_ for the tgt
// side
AMIPtr_->calculate(dummyPatch, patch_, surfPtr());
}
// Now the AMI addressing/weights will be from src side (on masterWorld
// processors) to tgt side (on other processors)
Pstream::warnComm = oldComm;
Pstream::worldComm = oldWorldComm;
......
......@@ -492,9 +492,12 @@ public:
//- Communicator
inline label comm() const;
//- Is world the local world
//- Is sample world the local world?
inline bool sameWorld() const;
//- Is my world ordered before the sampleWorld?
inline bool masterWorld() const;
//- Cached sampleRegion != mesh.name()
inline bool sameRegion() const;
......
......@@ -158,6 +158,23 @@ inline bool Foam::mappedPatchBase::sameWorld() const
}
inline bool Foam::mappedPatchBase::masterWorld() const
{
if (sameWorld())
{
return true;
}
else
{
// Use ordering in allWorlds
const label myWorld = UPstream::myWorldID();
const label mySampleWorld =
UPstream::allWorlds().find(sampleWorld_);
return myWorld < mySampleWorld;
}
}
inline bool Foam::mappedPatchBase::sameRegion() const
{