diff --git a/src/petsc4Foam/Make/options b/src/petsc4Foam/Make/options index a4d0a735fe47159a15d407563fdc707ce8e07219..22de33eb66aebf646c9ddc29a628902a94ec2fe1 100644 --- a/src/petsc4Foam/Make/options +++ b/src/petsc4Foam/Make/options @@ -13,6 +13,7 @@ EXE_INC = \ -Wno-old-style-cast \ -I$(LIB_SRC)/finiteVolume/lnInclude \ -I$(LIB_SRC)/meshTools/lnInclude \ + -I$(LIB_SRC)/Pstream/mpi \ $(foreach dir,$(PETSC_INC_DIR),-I$(dir)) LIB_LIBS = \ diff --git a/src/petsc4Foam/solvers/petscSolver.C b/src/petsc4Foam/solvers/petscSolver.C index 8b4f4d7a9e9f04487cf148b24d2121b9d1a4cef8..77c372fa73c4e2ae65cbde0348de5f0b148dabf6 100644 --- a/src/petsc4Foam/solvers/petscSolver.C +++ b/src/petsc4Foam/solvers/petscSolver.C @@ -33,6 +33,7 @@ License #include "cyclicLduInterface.H" #include "cyclicAMILduInterface.H" #include "addToRunTimeSelectionTable.H" +#include "PstreamGlobals.H" #include "petscSolver.H" #include "petscControls.H" @@ -354,7 +355,7 @@ void Foam::petscSolver::buildKsp ) const { // Create parallel solver context - KSPCreate(PETSC_COMM_WORLD, &ksp); + KSPCreate(PetscObjectComm((PetscObject)Amat), &ksp); // Set the prefix for the options db (e.g. -eqn_p_) KSPSetOptionsPrefix(ksp, prefix_.c_str()); @@ -384,6 +385,17 @@ void Foam::petscSolver::computeMatAllocation label& maxLowNonZeroPerRow ) const { + // communicator for the matrix + // PETSc internally duplicates the communicator to + // avoid tag/attributes clashing + + const auto comm = + ( + PstreamGlobals::MPICommunicators_.empty() + ? PETSC_COMM_SELF + : PstreamGlobals::MPICommunicators_[matrix_.mesh().comm()] + ); + // traversal-based algorithm typedef List<PetscInt> integerList; @@ -398,7 +410,7 @@ void Foam::petscSolver::computeMatAllocation const label nrows_ = lduAddr.size(); // Create a petsc matrix - MatCreate(PETSC_COMM_WORLD, &Amat); + MatCreate(comm, &Amat); MatSetSizes(Amat, nrows_, nrows_, PETSC_DECIDE, PETSC_DECIDE); // Set the prefix for the options db (e.g. -eqn_p_)