Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Development
openfoam
Commits
89f2cda3
Commit
89f2cda3
authored
Dec 08, 2020
by
mattijs
Committed by
Andrew Heather
Dec 09, 2020
Browse files
ENH: mpi: use per-application communicator.
parent
627d79db
Changes
42
Expand all
Hide whitespace changes
Inline
Side-by-side
applications/utilities/parallelProcessing/redistributePar/redistributePar.C
View file @
89f2cda3
...
...
@@ -511,9 +511,9 @@ void writeProcAddressing
patchDistMap
.
constructMap
(),
patchDistMap
.
constructHasFlip
(),
patchMap
,
label
(
-
1
),
eqOp
<
label
>
(),
flipOp
(),
label
(
-
1
),
UPstream
::
msgType
()
);
}
...
...
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
View file @
89f2cda3
...
...
@@ -90,8 +90,8 @@ void Foam::UPstream::setParRun(const label nProcs, const bool haveThreads)
<<
Foam
::
exit
(
FatalError
);
}
Pout
.
prefix
()
=
'['
+
name
(
myProcNo
(
Pstream
::
worldC
omm
))
+
"] "
;
Perr
.
prefix
()
=
'['
+
name
(
myProcNo
(
Pstream
::
worldC
omm
))
+
"] "
;
Pout
.
prefix
()
=
'['
+
name
(
myProcNo
(
c
omm
))
+
"] "
;
Perr
.
prefix
()
=
'['
+
name
(
myProcNo
(
c
omm
))
+
"] "
;
}
}
...
...
@@ -373,6 +373,8 @@ Foam::DynamicList<Foam::List<int>> Foam::UPstream::procIDs_(10);
Foam
::
DynamicList
<
Foam
::
label
>
Foam
::
UPstream
::
parentCommunicator_
(
10
);
Foam
::
wordList
Foam
::
UPstream
::
allWorlds_
(
1
,
""
);
Foam
::
labelList
Foam
::
UPstream
::
worldIDs_
(
1
,
0
);
Foam
::
DynamicList
<
Foam
::
List
<
Foam
::
UPstream
::
commsStruct
>>
Foam
::
UPstream
::
linearCommunication_
(
10
);
...
...
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View file @
89f2cda3
...
...
@@ -194,6 +194,13 @@ private:
//- Standard transfer message type
static
int
msgType_
;
//- Names of all worlds
static
wordList
allWorlds_
;
//- Per processor the name of the world
static
labelList
worldIDs_
;
// Communicator specific data
//- Free communicators
...
...
@@ -437,7 +444,7 @@ public:
}
//- Number of processes in parallel run, and 1 for serial run
static
label
nProcs
(
const
label
communicator
=
0
)
static
label
nProcs
(
const
label
communicator
=
worldComm
)
{
return
procIDs_
[
communicator
].
size
();
}
...
...
@@ -449,13 +456,13 @@ public:
}
//- Am I the master process
static
bool
master
(
const
label
communicator
=
0
)
static
bool
master
(
const
label
communicator
=
worldComm
)
{
return
myProcNo_
[
communicator
]
==
masterNo
();
}
//- Number of this process (starting from masterNo() = 0)
static
int
myProcNo
(
const
label
communicator
=
0
)
static
int
myProcNo
(
const
label
communicator
=
worldComm
)
{
return
myProcNo_
[
communicator
];
}
...
...
@@ -471,15 +478,43 @@ public:
return
procIDs_
[
communicator
];
}
// Worlds
//- All worlds
static
const
wordList
&
allWorlds
()
{
return
allWorlds_
;
}
//- worldID (index in allWorlds) of all processes
static
const
labelList
&
worldIDs
()
{
return
worldIDs_
;
}
//- My worldID
static
label
myWorldID
()
{
return
worldIDs_
[
myProcNo
(
0
)];
}
//- My world
static
const
word
&
myWorld
()
{
return
allWorlds
()[
myWorldID
()];
}
//- Range of process indices for all processes
static
rangeType
allProcs
(
const
label
communicator
=
0
)
static
rangeType
allProcs
(
const
label
communicator
=
worldComm
)
{
// Proc 0 -> nProcs (int value)
return
rangeType
(
static_cast
<
int
>
(
nProcs
(
communicator
)));
}
//- Range of process indices for sub-processes
static
rangeType
subProcs
(
const
label
communicator
=
0
)
static
rangeType
subProcs
(
const
label
communicator
=
worldComm
)
{
// Proc 1 -> nProcs (int value)
return
rangeType
(
1
,
static_cast
<
int
>
(
nProcs
(
communicator
)
-
1
));
...
...
@@ -488,7 +523,7 @@ public:
//- Communication schedule for linear all-to-master (proc 0)
static
const
List
<
commsStruct
>&
linearCommunication
(
const
label
communicator
=
0
const
label
communicator
=
worldComm
)
{
return
linearCommunication_
[
communicator
];
...
...
@@ -497,7 +532,7 @@ public:
//- Communication schedule for tree all-to-master (proc 0)
static
const
List
<
commsStruct
>&
treeCommunication
(
const
label
communicator
=
0
const
label
communicator
=
worldComm
)
{
return
treeCommunication_
[
communicator
];
...
...
@@ -542,7 +577,7 @@ public:
(
const
labelUList
&
sendData
,
labelUList
&
recvData
,
const
label
communicator
=
0
const
label
communicator
=
worldComm
);
//- Exchange data with all processors (in the communicator)
...
...
@@ -559,7 +594,7 @@ public:
const
UList
<
int
>&
recvSizes
,
const
UList
<
int
>&
recvOffsets
,
const
label
communicator
=
0
const
label
communicator
=
worldComm
);
//- Receive data from all processors on the master
...
...
@@ -571,7 +606,7 @@ public:
char
*
recvData
,
const
UList
<
int
>&
recvSizes
,
const
UList
<
int
>&
recvOffsets
,
const
label
communicator
=
0
const
label
communicator
=
worldComm
);
//- Send data to all processors from the root of the communicator
...
...
@@ -583,7 +618,7 @@ public:
char
*
recvData
,
int
recvSize
,
const
label
communicator
=
0
const
label
communicator
=
worldComm
);
...
...
@@ -598,7 +633,7 @@ public:
//- Process index of last sub-process
// \deprecated(2020-09) use subProcs() method instead
static
int
lastSlave
(
const
label
communicator
=
0
)
static
int
lastSlave
(
const
label
communicator
=
worldComm
)
{
return
nProcs
(
communicator
)
-
1
;
}
...
...
src/OpenFOAM/global/argList/argList.C
View file @
89f2cda3
...
...
@@ -166,6 +166,20 @@ Foam::argList::initValidTables::initValidTables()
true
// advanced option
);
argList
::
addOption
(
"world"
,
"Name"
,
"Name of local world"
,
true
);
validParOptions
.
set
(
"world"
,
"Name of local world"
);
// Some standard option aliases (with or without version warnings)
// argList::addOptionCompat
// (
...
...
@@ -493,6 +507,7 @@ void Foam::argList::noParallel()
removeOption
(
"roots"
);
removeOption
(
"decomposeParDict"
);
removeOption
(
"hostRoots"
);
removeOption
(
"world"
);
validParOptions
.
clear
();
}
...
...
@@ -711,6 +726,7 @@ void Foam::argList::setCasePaths()
}
else
{
caseDir
.
expand
();
caseDir
.
toAbsolute
();
}
}
...
...
@@ -1241,7 +1257,7 @@ void Foam::argList::parse
dictNProcs
=
roots
.
size
()
+
1
;
}
}
else
if
(
checkProcessorDirectories_
)
else
if
(
checkProcessorDirectories_
&&
Pstream
::
nProcs
()
>
1
)
{
// Use values from decomposeParDict, the location was already
// established above.
...
...
@@ -1268,6 +1284,13 @@ void Foam::argList::parse
Pstream
::
parRun
(
oldParRun
);
// Restore parallel state
decompDict
.
readEntry
(
"numberOfSubdomains"
,
dictNProcs
);
if
(
Pstream
::
nProcs
()
==
1
)
{
WarningInFunction
<<
"Running parallel on single processor. This only"
<<
" makes sense for multi-world simulation"
<<
endl
;
dictNProcs
=
1
;
}
if
(
decompDict
.
getOrDefault
(
"distributed"
,
false
))
{
...
...
@@ -1300,7 +1323,12 @@ void Foam::argList::parse
// - normal running : nProcs = dictNProcs = nProcDirs
// - decomposition to more processors : nProcs = dictNProcs
// - decomposition to fewer processors : nProcs = nProcDirs
if
(
checkProcessorDirectories_
&&
dictNProcs
>
Pstream
::
nProcs
())
if
(
checkProcessorDirectories_
&&
Pstream
::
nProcs
()
>
1
&&
dictNProcs
>
Pstream
::
nProcs
()
)
{
FatalError
<<
source
...
...
@@ -1353,6 +1381,7 @@ void Foam::argList::parse
if
(
checkProcessorDirectories_
&&
Pstream
::
nProcs
()
>
1
&&
dictNProcs
<
Pstream
::
nProcs
()
)
{
...
...
@@ -1405,7 +1434,14 @@ void Foam::argList::parse
}
nProcs
=
Pstream
::
nProcs
();
case_
=
globalCase_
/
(
"processor"
+
Foam
::
name
(
Pstream
::
myProcNo
()));
if
(
Pstream
::
nProcs
()
>
1
)
{
case_
=
globalCase_
/
(
"processor"
+
Foam
::
name
(
Pstream
::
myProcNo
()));
}
else
{
case_
=
globalCase_
;
}
}
else
{
...
...
@@ -1468,6 +1504,13 @@ void Foam::argList::parse
<<
Pstream
::
commsTypeNames
[
Pstream
::
defaultCommsType
]
<<
nl
<<
" polling iterations : "
<<
Pstream
::
nPollProcInterfaces
<<
endl
;
if
(
UPstream
::
allWorlds
().
size
()
>
1
)
{
Info
<<
" worlds : "
<<
flatOutput
(
UPstream
::
allWorlds
())
<<
nl
<<
" world : "
<<
UPstream
::
myWorld
()
<<
endl
;
}
}
}
...
...
src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexI.H
View file @
89f2cda3
...
...
@@ -93,7 +93,7 @@ inline Foam::label Foam::globalIndex::size() const
inline
void
Foam
::
globalIndex
::
reset
(
const
label
localSize
)
{
reset
(
localSize
,
Pstream
::
msgType
(),
0
,
true
);
reset
(
localSize
,
Pstream
::
msgType
(),
UPstream
::
worldComm
,
true
);
}
...
...
src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C
View file @
89f2cda3
...
...
@@ -2745,13 +2745,16 @@ void Foam::globalMeshData::updateMesh()
// *** Temporary hack to avoid problems with overlapping communication
// *** between these reductions and the calculation of deltaCoeffs
//label comm = UPstream::worldComm + 1;
label
comm
=
UPstream
::
allocateCommunicator
//
const
label comm = UPstream::worldComm + 1;
const
label
comm
=
UPstream
::
allocateCommunicator
(
UPstream
::
worldComm
,
identity
(
UPstream
::
nProcs
(
UPstream
::
worldComm
)),
true
);
const
label
oldWarnComm
=
UPstream
::
warnComm
;
UPstream
::
warnComm
=
comm
;
// Total number of faces.
nTotalFaces_
=
returnReduce
...
...
@@ -2789,6 +2792,7 @@ void Foam::globalMeshData::updateMesh()
);
UPstream
::
freeCommunicator
(
comm
);
UPstream
::
warnComm
=
oldWarnComm
;
if
(
debug
)
{
...
...
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
View file @
89f2cda3
...
...
@@ -148,9 +148,9 @@ void Foam::mapDistribute::printLayout(Ostream& os) const
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam
::
mapDistribute
::
mapDistribute
()
Foam
::
mapDistribute
::
mapDistribute
(
const
label
comm
)
:
mapDistributeBase
()
mapDistributeBase
(
comm
)
{}
...
...
@@ -176,7 +176,8 @@ Foam::mapDistribute::mapDistribute
labelListList
&&
subMap
,
labelListList
&&
constructMap
,
const
bool
subHasFlip
,
const
bool
constructHasFlip
const
bool
constructHasFlip
,
const
label
comm
)
:
mapDistributeBase
...
...
@@ -185,7 +186,8 @@ Foam::mapDistribute::mapDistribute
std
::
move
(
subMap
),
std
::
move
(
constructMap
),
subHasFlip
,
constructHasFlip
constructHasFlip
,
comm
)
{}
...
...
@@ -198,7 +200,8 @@ Foam::mapDistribute::mapDistribute
labelListList
&&
transformElements
,
labelList
&&
transformStart
,
const
bool
subHasFlip
,
const
bool
constructHasFlip
const
bool
constructHasFlip
,
const
label
comm
)
:
mapDistributeBase
...
...
@@ -207,7 +210,8 @@ Foam::mapDistribute::mapDistribute
std
::
move
(
subMap
),
std
::
move
(
constructMap
),
subHasFlip
,
constructHasFlip
constructHasFlip
,
comm
),
transformElements_
(
std
::
move
(
transformElements
)),
transformStart_
(
std
::
move
(
transformStart
))
...
...
@@ -217,10 +221,11 @@ Foam::mapDistribute::mapDistribute
Foam
::
mapDistribute
::
mapDistribute
(
const
labelUList
&
sendProcs
,
const
labelUList
&
recvProcs
const
labelUList
&
recvProcs
,
const
label
comm
)
:
mapDistributeBase
(
sendProcs
,
recvProcs
)
mapDistributeBase
(
sendProcs
,
recvProcs
,
comm
)
{}
...
...
@@ -229,7 +234,8 @@ Foam::mapDistribute::mapDistribute
const
globalIndex
&
globalNumbering
,
labelList
&
elements
,
List
<
Map
<
label
>>&
compactMap
,
const
int
tag
const
int
tag
,
const
label
comm
)
:
mapDistributeBase
...
...
@@ -237,7 +243,8 @@ Foam::mapDistribute::mapDistribute
globalNumbering
,
elements
,
compactMap
,
tag
tag
,
comm
)
{}
...
...
@@ -247,7 +254,8 @@ Foam::mapDistribute::mapDistribute
const
globalIndex
&
globalNumbering
,
labelListList
&
cellCells
,
List
<
Map
<
label
>>&
compactMap
,
const
int
tag
const
int
tag
,
const
label
comm
)
:
mapDistributeBase
...
...
@@ -255,7 +263,8 @@ Foam::mapDistribute::mapDistribute
globalNumbering
,
cellCells
,
compactMap
,
tag
tag
,
comm
)
{}
...
...
@@ -268,11 +277,14 @@ Foam::mapDistribute::mapDistribute
const
labelPairList
&
transformedElements
,
labelList
&
transformedIndices
,
List
<
Map
<
label
>>&
compactMap
,
const
int
tag
const
int
tag
,
const
label
comm
)
:
mapDistributeBase
()
mapDistributeBase
(
comm
)
{
const
label
myRank
=
Pstream
::
myProcNo
(
comm
);
// Construct per processor compact addressing of the global elements
// needed. The ones from the local processor are not included since
// these are always all needed.
...
...
@@ -288,7 +300,7 @@ Foam::mapDistribute::mapDistribute
{
labelPair
elem
=
transformedElements
[
i
];
label
proci
=
globalTransforms
.
processor
(
elem
);
if
(
proci
!=
Pstream
::
myProcNo
()
)
if
(
proci
!=
myRank
)
{
label
index
=
globalTransforms
.
index
(
elem
);
label
nCompact
=
compactMap
[
proci
].
size
();
...
...
@@ -345,7 +357,7 @@ Foam::mapDistribute::mapDistribute
// Get compact index for untransformed element
label
rawElemI
=
(
proci
==
Pstream
::
myProcNo
()
proci
==
myRank
?
index
:
compactMap
[
proci
][
index
]
);
...
...
@@ -373,11 +385,14 @@ Foam::mapDistribute::mapDistribute
const
List
<
labelPairList
>&
transformedElements
,
labelListList
&
transformedIndices
,
List
<
Map
<
label
>>&
compactMap
,
const
int
tag
const
int
tag
,
const
label
comm
)
:
mapDistributeBase
()
mapDistributeBase
(
comm
)
{
const
label
myRank
=
Pstream
::
myProcNo
(
comm_
);
// Construct per processor compact addressing of the global elements
// needed. The ones from the local processor are not included since
// these are always all needed.
...
...
@@ -396,7 +411,7 @@ Foam::mapDistribute::mapDistribute
forAll
(
elems
,
i
)
{
label
proci
=
globalTransforms
.
processor
(
elems
[
i
]);
if
(
proci
!=
Pstream
::
myProcNo
()
)
if
(
proci
!=
myRank
)
{
label
index
=
globalTransforms
.
index
(
elems
[
i
]);
label
nCompact
=
compactMap
[
proci
].
size
();
...
...
@@ -462,7 +477,7 @@ Foam::mapDistribute::mapDistribute
// Get compact index for untransformed element
label
rawElemI
=
(
proci
==
Pstream
::
myProcNo
()
proci
==
myRank
?
index
:
compactMap
[
proci
][
index
]
);
...
...
@@ -487,10 +502,11 @@ Foam::mapDistribute::mapDistribute
(
labelListList
&&
subMap
,
const
bool
subHasFlip
,
const
bool
constructHasFlip
const
bool
constructHasFlip
,
const
label
comm
)
:
mapDistributeBase
(
std
::
move
(
subMap
),
subHasFlip
,
constructHasFlip
)
mapDistributeBase
(
std
::
move
(
subMap
),
subHasFlip
,
constructHasFlip
,
comm
)
{}
...
...
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
View file @
89f2cda3
...
...
@@ -328,7 +328,7 @@ public:
// Constructors
//- Construct null
mapDistribute
();
mapDistribute
(
const
label
comm
=
UPstream
::
worldComm
);
//- Copy construct
explicit
mapDistribute
(
const
mapDistribute
&
map
);
...
...
@@ -343,7 +343,8 @@ public:
labelListList
&&
subMap
,
labelListList
&&
constructMap
,
const
bool
subHasFlip
=
false
,
const
bool
constructHasFlip
=
false
const
bool
constructHasFlip
=
false
,
const
label
comm
=
UPstream
::
worldComm
);
//- Move construct from components
...
...
@@ -355,7 +356,8 @@ public:
labelListList
&&
transformElements
,
labelList
&&
transformStart
,
const
bool
subHasFlip
=
false
,
const
bool
constructHasFlip
=
false
const
bool
constructHasFlip
=
false
,
const
label
comm
=
UPstream
::
worldComm
);
//- Construct from reverse addressing: per data item the send
...
...
@@ -365,7 +367,8 @@ public:
mapDistribute
(
const
labelUList
&
sendProcs
,
const
labelUList
&
recvProcs
const
labelUList
&
recvProcs
,
const
label
comm
=
UPstream
::
worldComm
);
//- Construct from list of (possibly) remote elements in globalIndex
...
...
@@ -378,7 +381,8 @@ public:
const
globalIndex
&
,
labelList
&
elements
,
List
<
Map
<
label
>>&
compactMap
,
const
int
tag
=
Pstream
::
msgType
()
const
int
tag
=
Pstream
::
msgType
(),
const
label
comm
=
UPstream
::
worldComm
);
//- Special variant that works with the info sorted into bins
...
...
@@ -390,7 +394,8 @@ public:
const
globalIndex
&
,
labelListList
&
cellCells
,
List
<
Map
<
label
>>&
compactMap
,
const
int
tag
=
Pstream
::
msgType
()
const
int
tag
=
Pstream
::
msgType
(),
const
label
comm
=
UPstream
::
worldComm
);
//- Construct from list of (possibly remote) untransformed elements
...
...
@@ -407,7 +412,8 @@ public:
const
labelPairList
&
transformedElements
,
labelList
&
transformedIndices
,
List
<
Map
<
label
>>&
compactMap
,
const
int
tag
=
Pstream
::
msgType
()
const
int
tag
=
Pstream
::
msgType
(),
const
label
comm
=
UPstream
::
worldComm
);
//- As above but with ListLists.
...
...
@@ -419,7 +425,8 @@ public:
const
List
<
labelPairList
>&
transformedElements
,
labelListList
&
transformedIndices
,
List
<
Map
<
label
>>&
compactMap
,
const
int
tag
=
Pstream
::
msgType
()
const
int
tag
=
Pstream
::
msgType
(),
const
label
comm
=
UPstream
::
worldComm
);
//- Construct from my elements to send. Assumes layout is my elements
...
...
@@ -429,7 +436,8 @@ public:
(
labelListList
&&
subMap
,
const
bool
subHasFlip
=
false
,
const
bool
constructHasFlip
=
false
const
bool
constructHasFlip
=
false
,
const
label
comm
=
UPstream
::
worldComm
);
//- Construct from Istream
...
...
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C
View file @
89f2cda3
This diff is collapsed.
Click to expand it.
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.H
View file @
89f2cda3
...
...
@@ -122,6 +122,8 @@ protected:
//- Whether constructMap includes flip or not
bool
constructHasFlip_
;
//- Communicator to use for parallel operations
label
comm_
;
//- Schedule
mutable
autoPtr
<
List
<
labelPair
>>
schedulePtr_
;
...
...
@@ -199,7 +201,7 @@ public:
// Constructors
//- Construct null
mapDistributeBase
();
mapDistributeBase
(
const
label
comm
=
UPstream
::
worldComm
);
//- Copy construct
mapDistributeBase
(
const
mapDistributeBase
&
map
);
...
...
@@ -214,7 +216,8 @@ public:
labelListList
&&
subMap
,
labelListList
&&
constructMap
,