diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index 81562f699817431fb484f082643dd0201afc820b..c840c89e1a33df5046d620311b48263b2ce906fa 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -257,10 +257,13 @@ Foam::label Foam::UPstream::allocateCommunicator treeCommunication_.append(List<commsStruct>(0)); } - Pout<< "Communicators : Allocating communicator " << index << endl - << " parent : " << parentIndex << endl - << " procs : " << subRanks << endl - << endl; + if (debug) + { + Pout<< "Communicators : Allocating communicator " << index << endl + << " parent : " << parentIndex << endl + << " procs : " << subRanks << endl + << endl; + } // Initialise; overwritten by allocatePstreamCommunicator myProcNo_[index] = 0; @@ -305,10 +308,13 @@ void Foam::UPstream::freeCommunicator const bool doPstream ) { - Pout<< "Communicators : Freeing communicator " << communicator << endl - << " parent : " << parentCommunicator_[communicator] << endl - << " myProcNo : " << myProcNo_[communicator] << endl - << endl; + if (debug) + { + Pout<< "Communicators : Freeing communicator " << communicator << endl + << " parent : " << parentCommunicator_[communicator] << endl + << " myProcNo : " << myProcNo_[communicator] << endl + << endl; + } if (doPstream && parRun()) { @@ -326,9 +332,6 @@ void Foam::UPstream::freeCommunicator void Foam::UPstream::freeCommunicators(const bool doPstream) { - Pout<< "Communicators : Freeing all communicators" << endl - << endl; - forAll(myProcNo_, communicator) { if (myProcNo_[communicator] != -1) diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index f23ee07bfc13346076a4f6b04601258c89624f6a..dc4d142c260cf93ceed333bc13348c2c2b2f6a1a 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -384,12 +384,6 @@ public: static void freeTag(const word&, const int tag); -// //- Return (index of) request to use in calls to nonblocking ops -// static label allocateRequest(const word&); -// -// //- Free (index of) request -// static void freeRequest(const word&, const label request); - //- Is this a parallel run? static bool& parRun() @@ -426,13 +420,6 @@ public: return parentCommunicator_(communicator); } -// //- Process IDs -// static const List<int>& procIDs() -// { -// return procIDs_; -// } - - //- Process ID of given process index static List<int>& procID(int communicator) { diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.C index 2fb1cbe39b068ea43f3c66eeac4ecf7ddc27da3b..c47c0aeee6ef82b7c420b40450a1dfaccd71c4a8 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.C @@ -91,19 +91,12 @@ Foam::GAMGSolver::GAMGSolver { if (agglomeration_.hasMeshLevel(fineLevelIndex)) { - Pout<< "Level:" << fineLevelIndex - << " agglomerating matrix." << endl; - if ( (fineLevelIndex+1) < agglomeration_.size() - //&& agglomeration_.procBoundaryMap_.set(fineLevelIndex+1) && agglomeration_.hasProcMesh(fineLevelIndex+1) ) { - Pout<< "Level:" << fineLevelIndex - << " agglomerating onto dummy coarse mesh." << endl; - // Construct matrix without referencing the coarse mesh so // construct a dummy mesh instead. This will get overwritten // by the call to procAgglomerateMatrix so is only to get @@ -174,36 +167,21 @@ Foam::GAMGSolver::GAMGSolver const List<int>& procIDs = agglomeration_.agglomProcIDs(fineLevelIndex+1); - Pout<< "Level:" << fineLevelIndex - << " agglomerating onto procIDs:" << procIDs << endl; - procAgglomerateMatrix ( procAgglomMap, procIDs, fineLevelIndex ); - - Pout<< "Level:" << fineLevelIndex - << " DONE agglomerating onto procIDs:" << procIDs - << endl; } else { - Pout<< "Level:" << fineLevelIndex - << " agglomerating onto coarse mesh at level " - << fineLevelIndex + 1 << endl; - agglomerateMatrix ( fineLevelIndex, agglomeration_.meshLevel(fineLevelIndex + 1), agglomeration_.interfaceLevel(fineLevelIndex + 1) ); - Pout<< "Level:" << fineLevelIndex - << " DONE agglomerating onto coarse mesh at level " - << fineLevelIndex + 1 << endl; - } } else @@ -273,9 +251,6 @@ Foam::GAMGSolver::GAMGSolver { const label coarsestLevel = matrixLevels_.size() - 1; - Pout<< "GAMGSolver :" - << " coarsestLevel:" << coarsestLevel << endl; - if (matrixLevels_.set(coarsestLevel)) { const lduMesh& coarsestMesh = @@ -285,9 +260,6 @@ Foam::GAMGSolver::GAMGSolver label oldWarn = UPstream::warnComm; UPstream::warnComm = coarseComm; - Pout<< "Solve direct on coasestmesh (level=" << coarsestLevel - << ") using communicator " << coarseComm << endl; - coarsestLUMatrixPtr_.set ( new LUscalarMatrix @@ -301,119 +273,6 @@ Foam::GAMGSolver::GAMGSolver UPstream::warnComm = oldWarn; } } - //else if (agglomeration_.processorAgglomerate()) - //{ - //// Pick a level to processor agglomerate - //label agglomLevel = matrixLevels_.size() - 1;//1; - // - // - //// Get mesh and matrix at this level - //const lduMatrix& levelMatrix = matrixLevels_[agglomLevel]; - //const lduMesh& levelMesh = levelMatrix.mesh(); - // - // - //label levelComm = levelMesh.comm(); - //label oldWarn = UPstream::warnComm; - //UPstream::warnComm = levelComm; - // - //Pout<< "Solve generic on mesh (level=" << agglomLevel - // << ") using communicator " << levelComm << endl; - // - //// Processor restriction map: per processor the coarse processor - //labelList procAgglomMap(UPstream::nProcs(levelComm)); - //// Master processor - //labelList masterProcs; - //// Local processors that agglomerate. agglomProcIDs[0] is in - //// masterProc. - //List<int> agglomProcIDs; - // - //{ - // procAgglomMap[0] = 0; - // procAgglomMap[1] = 0; - // procAgglomMap[2] = 1; - // procAgglomMap[3] = 1; - // - // // Determine the master processors - // Map<label> agglomToMaster(procAgglomMap.size()); - // - // forAll(procAgglomMap, procI) - // { - // label coarseI = procAgglomMap[procI]; - // - // Map<label>::iterator fnd = agglomToMaster.find(coarseI); - // if (fnd == agglomToMaster.end()) - // { - // agglomToMaster.insert(coarseI, procI); - // } - // else - // { - // fnd() = max(fnd(), procI); - // } - // } - // - // masterProcs.setSize(agglomToMaster.size()); - // forAllConstIter(Map<label>, agglomToMaster, iter) - // { - // masterProcs[iter.key()] = iter(); - // } - // - // - // // Collect all the processors in my agglomeration - // label myProcID = Pstream::myProcNo(levelComm); - // label myAgglom = procAgglomMap[myProcID]; - // - // // Get all processors agglomerating to the same coarse - // // processor - // agglomProcIDs = findIndices(procAgglomMap, myAgglom); - // // Make sure the master is the first element. - // label index = findIndex - // ( - // agglomProcIDs, - // agglomToMaster[myAgglom] - // ); - // Swap(agglomProcIDs[0], agglomProcIDs[index]); - //} - // - // - //Pout<< "procAgglomMap:" << procAgglomMap << endl; - //Pout<< "agglomProcIDs:" << agglomProcIDs << endl; - // - //// Allocate a communicator for the processor-agglomerated matrix - //label procAgglomComm = UPstream::allocateCommunicator - //( - // levelComm, - // masterProcs - //); - //Pout<< "** Allocated communicator " << procAgglomComm - // << " for indices " << masterProcs - // << " in processor list " << UPstream::procID(levelComm) - // << endl; - // - // - // - //// Gather matrix and mesh onto agglomProcIDs[0] - //// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - // - //procAgglomerateMatrix - //( - // // Agglomeration information - // procAgglomMap, - // agglomProcIDs, - // procAgglomComm, - // - // agglomLevel, // level (coarse, not fine level!) - // - // // Resulting matrix - // allMatrixPtr_, - // allInterfaceBouCoeffs_, - // allInterfaceIntCoeffs_, - // allPrimitiveInterfaces_, - // allInterfaces_ - //); - // - // - //UPstream::warnComm = oldWarn; - //} } else { @@ -441,20 +300,6 @@ Foam::GAMGSolver::GAMGSolver Foam::GAMGSolver::~GAMGSolver() { -// // Clear the the lists of pointers to the interfaces -// forAll(interfaceLevels_, leveli) -// { -// lduInterfaceFieldPtrsList& curLevel = interfaceLevels_[leveli]; -// -// forAll(curLevel, i) -// { -// if (curLevel.set(i)) -// { -// delete curLevel(i); -// } -// } -// } - if (!cacheAgglomeration_) { delete &agglomeration_; @@ -489,19 +334,22 @@ void Foam::GAMGSolver::readControls() controlDict_.readIfPresent("scaleCorrection", scaleCorrection_); controlDict_.readIfPresent("directSolveCoarsest", directSolveCoarsest_); - Pout<< "GAMGSolver settings :" - << " cacheAgglomeration:" << cacheAgglomeration_ - << " nPreSweeps:" << nPreSweeps_ - << " preSweepsLevelMultiplier:" << preSweepsLevelMultiplier_ - << " maxPreSweeps:" << maxPreSweeps_ - << " nPostSweeps:" << nPostSweeps_ - << " postSweepsLevelMultiplier:" << postSweepsLevelMultiplier_ - << " maxPostSweeps:" << maxPostSweeps_ - << " nFinestSweeps:" << nFinestSweeps_ - << " interpolateCorrection:" << interpolateCorrection_ - << " scaleCorrection:" << scaleCorrection_ - << " directSolveCoarsest:" << directSolveCoarsest_ - << endl; + if (debug) + { + Pout<< "GAMGSolver settings :" + << " cacheAgglomeration:" << cacheAgglomeration_ + << " nPreSweeps:" << nPreSweeps_ + << " preSweepsLevelMultiplier:" << preSweepsLevelMultiplier_ + << " maxPreSweeps:" << maxPreSweeps_ + << " nPostSweeps:" << nPostSweeps_ + << " postSweepsLevelMultiplier:" << postSweepsLevelMultiplier_ + << " maxPostSweeps:" << maxPostSweeps_ + << " nFinestSweeps:" << nFinestSweeps_ + << " interpolateCorrection:" << interpolateCorrection_ + << " scaleCorrection:" << scaleCorrection_ + << " directSolveCoarsest:" << directSolveCoarsest_ + << endl; + } } diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C index 9f40158872d7428d7143972204576dd591fc91d8..dc3cad105187641fb5d97c942fd515890016eeb6 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C @@ -45,14 +45,6 @@ void Foam::GAMGSolver::agglomerateMatrix const label nCoarseFaces = agglomeration_.nFaces(fineLevelIndex); const label nCoarseCells = agglomeration_.nCells(fineLevelIndex); - - Pout<< "agglomerateMatrix : I have fine mesh:" - << fineMatrix.mesh().lduAddr().size() - << " at fine level:" << fineLevelIndex - << " constricting to coarse cells:" - << nCoarseCells - << " coarse nFaces:" << nCoarseFaces << endl; - // Set the coarse level matrix matrixLevels_.set ( @@ -307,18 +299,16 @@ void Foam::GAMGSolver::gatherMatrices List<List<int> >& otherRanks ) const { - Pout<< "GAMGSolver::gatherMatrices :" - << " collecting matrices on procs:" << procIDs - << " using comm:" << meshComm << endl; + if (debug) + { + Pout<< "GAMGSolver::gatherMatrices :" + << " collecting matrices from procs:" << procIDs + << " using comm:" << meshComm << endl; + } if (Pstream::myProcNo(meshComm) == procIDs[0]) { // Master. - - Pout<< "GAMGSolver::gatherMatrices :" - << " master:" << Pstream::myProcNo(meshComm) << endl; - - otherMats.setSize(procIDs.size()-1); otherBouCoeffs.setSize(procIDs.size()-1); otherIntCoeffs.setSize(procIDs.size()-1); @@ -378,10 +368,6 @@ void Foam::GAMGSolver::gatherMatrices } else { - //Pout<< "GAMGSolver::gatherMatrices :" - // << " sending from:" << Pstream::myProcNo(meshComm) - // << " to master:" << procIDs[0] << endl; - // Send to master // Count valid interfaces @@ -402,9 +388,6 @@ void Foam::GAMGSolver::gatherMatrices } } - //Pout<< "GAMGSolver::gatherMatrices :" - // << " sending matrix:" << mat.info() << endl; - OPstream toMaster ( Pstream::scheduled, @@ -460,9 +443,6 @@ void Foam::GAMGSolver::procAgglomerateMatrix UPstream::warnComm = coarseComm; - // Construct (on the agglomeration) a complete mesh with mapping - Pout<< "procAgglomerateMatrix :" << " level:" << levelI << endl; - // Gather all matrix coefficients onto agglomProcIDs[0] // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -496,18 +476,17 @@ void Foam::GAMGSolver::procAgglomerateMatrix // Agglomerate all matrix // ~~~~~~~~~~~~~~~~~~~~~~ - Pout<< "Own matrix:" << coarsestMatrix.info() << endl; - - forAll(otherMats, i) - { - Pout<< "** otherMats " << i << " " - << otherMats[i].info() - << endl; - } - Pout<< endl; + //Pout<< "Own matrix:" << coarsestMatrix.info() << endl; + // + //forAll(otherMats, i) + //{ + // Pout<< "** otherMats " << i << " " + // << otherMats[i].info() + // << endl; + //} + //Pout<< endl; - //const lduMesh& allMesh = agglomeration_.procMeshLevel(levelI); const lduMesh& allMesh = agglomeration_.meshLevel(levelI+1); const labelList& cellOffsets = agglomeration_.cellOffsets(levelI+1); const labelListList& faceMap = agglomeration_.faceMap(levelI+1); @@ -515,8 +494,6 @@ void Foam::GAMGSolver::procAgglomerateMatrix const labelListListList& boundaryFaceMap = agglomeration_.boundaryFaceMap(levelI+1); -Pout<< "Agglomerating onto mesh:" << allMesh.info() << endl; - allMatrixPtr.reset(new lduMatrix(allMesh)); lduMatrix& allMatrix = allMatrixPtr(); @@ -679,10 +656,6 @@ Pout<< "Agglomerating onto mesh:" << allMesh.info() << endl; const labelList& map = boundaryFaceMap[procI][procIntI]; - //Pout<< " from proc:" << procI - //<< " interface:" << procIntI - //<< " mapped to faces:" << map << endl; - const scalarField& procBou = procBouCoeffs[procIntI]; const scalarField& procInt = procIntCoeffs[procIntI]; @@ -703,11 +676,6 @@ Pout<< "Agglomerating onto mesh:" << allMesh.info() << endl; // Boundary has become internal face const labelList& map = boundaryFaceMap[procI][procIntI]; - - //const labelList& allU = allMesh.lduAddr().upperAddr(); - //const labelList& allL = allMesh.lduAddr().lowerAddr(); - //const label off = cellOffsets[procI]; - const scalarField& procBou = procBouCoeffs[procIntI]; const scalarField& procInt = procIntCoeffs[procIntI]; @@ -740,72 +708,35 @@ Pout<< "Agglomerating onto mesh:" << allMesh.info() << endl; allMatrix.lower()[allFaceI] = -procBou[i]; } } - - - //// Simple check - //label allFaceI = - //( - // map[i] >= 0 - // ? map[i] - // : -map[i]-1 - //); - // - //const labelList& fCells = - // lduPrimitiveMesh::mesh - // ( - // coarsestMesh, - // agglomeration_.otherMeshes(), - // procI - // ).lduAddr().patchAddr(procIntI); - // - //label allCellI = off + fCells[i]; - // - //if - //( - // allCellI != allL[allFaceI] - // && allCellI != allU[allFaceI] - //) - //{ - // FatalErrorIn - // ( - // "GAMGSolver::GAMGSolver()" - // ) << "problem." - // << " allFaceI:" << allFaceI - // << " local cellI:" << fCells[i] - // << " allCellI:" << allCellI - // << " allLower:" << allL[allFaceI] - // << " allUpper:" << allU[allFaceI] - // << abort(FatalError); - //} } } } } - Pout<< "** Assembled allMatrix:" << allMatrix.info() << endl; - - forAll(allInterfaces, intI) - { - if (allInterfaces.set(intI)) - { - Pout<< " patch:" << intI - << " type:" << allInterfaces[intI].type() - << " size:" - << allInterfaces[intI].interface(). - faceCells().size() - << endl; - - //const scalarField& bouCoeffs = allInterfaceBouCoeffs[intI]; - //const scalarField& intCoeffs = allInterfaceIntCoeffs[intI]; - //forAll(bouCoeffs, faceI) - //{ - // Pout<< " " << faceI - // << "\tbou:" << bouCoeffs[faceI] - // << "\tint:" << intCoeffs[faceI] - // << endl; - //} - } - } + //Pout<< "** Assembled allMatrix:" << allMatrix.info() << endl; + // + //forAll(allInterfaces, intI) + //{ + // if (allInterfaces.set(intI)) + // { + // Pout<< " patch:" << intI + // << " type:" << allInterfaces[intI].type() + // << " size:" + // << allInterfaces[intI].interface(). + // faceCells().size() + // << endl; + // + // //const scalarField& bouCoeffs = allInterfaceBouCoeffs[intI]; + // //const scalarField& intCoeffs = allInterfaceIntCoeffs[intI]; + // //forAll(bouCoeffs, faceI) + // //{ + // // Pout<< " " << faceI + // // << "\tbou:" << bouCoeffs[faceI] + // // << "\tint:" << intCoeffs[faceI] + // // << endl; + // //} + // } + //} } UPstream::warnComm = oldWarn; } diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index 1b53aa1566e04c0b28f64c49e9807925a1fb1538..70b5d1923218e6db2f56d8c9ebecb1907c344059 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -327,10 +327,6 @@ void Foam::UPstream::allocatePstreamCommunicator { // Allocate world communicator - //std::cout - // << "MPI : Allocating world communicator at index " << index - // << std::endl; - if (index != UPstream::worldComm) { FatalErrorIn @@ -359,16 +355,6 @@ void Foam::UPstream::allocatePstreamCommunicator } else { - std::cout - << "MPI : Allocating new communicator at index " << index - << " from parent " << parentIndex - << std::endl; - for (label i=0; i < procIDs_[index].size(); i++) - { - std::cout<< " " << i << " rank:" << procIDs_[index][i] - << std::endl; - } - // Create new group MPI_Group_incl ( @@ -378,11 +364,6 @@ void Foam::UPstream::allocatePstreamCommunicator &PstreamGlobals::MPIGroups_[index] ); - //std::cout - // << "MPI : New group " << long(PstreamGlobals::MPIGroups_[index]) - // << std::endl; - - // Create new communicator MPI_Comm_create ( @@ -393,17 +374,10 @@ void Foam::UPstream::allocatePstreamCommunicator if (PstreamGlobals::MPICommunicators_[index] == MPI_COMM_NULL) { - //std::cout - // << "MPI : NULL : not in group" - // << std::endl; myProcNo_[index] = -1; } else { - std::cout - << "MPI : New comm " - << long(PstreamGlobals::MPICommunicators_[index]) - << std::endl; if ( MPI_Comm_rank @@ -429,9 +403,6 @@ void Foam::UPstream::allocatePstreamCommunicator } } } - - std::cout<< "MPI : in communicator " << index - << " I am rank " << myProcNo_[index] << std::endl; } @@ -683,62 +654,4 @@ void Foam::UPstream::freeTag(const word& s, const int tag) } -//Foam::label Foam::UPstream::allocateRequest(const word& s) -//{ -// label request; -// if (PstreamGlobals::freedRequests_.size()) -// { -// request = PstreamGlobals::freedRequests_.remove(); -// } -// else -// { -// request = PstreamGlobals::outstandingRequests_.size(); -// -// // Make sure outstanding requests table can hold this -// PstreamGlobals::outstandingRequests_.reserve(request); -// } -// -// if (debug) -// { -// //if (UPstream::lateBlocking > 0) -// //{ -// // string& poutp = Pout.prefix(); -// // poutp[poutp.size()-(UPstream::lateBlocking+2)+request] = 'X'; -// // Perr.prefix() = Pout.prefix(); -// //} -// Pout<< "UPstream::allocateRequest " << s -// << " : request:" << request -// << " out of:" << PstreamGlobals::outstandingRequests_.size() -// << " capacity:" << PstreamGlobals::outstandingRequests_.capacity() -// << endl; -// } -// -// return request; -//} -// -// -//void Foam::UPstream::freeRequest(const word& s, const label index) -//{ -// if (PstreamGlobals::outstandingRequests_(index) != MPI_REQUEST_NULL) -// { -// PstreamGlobals::outstandingRequests_(index) = MPI_REQUEST_NULL; -// PstreamGlobals::freedRequests_.append(index); -// -// if (debug) -// { -// //if (UPstream::lateBlocking > 0) -// //{ -// // string& poutp = Pout.prefix(); -// // poutp[poutp.size()-(UPstream::lateBlocking+2)+index] = ' '; -// // Perr.prefix() = Pout.prefix(); -// //} -// Pout<< "UPstream::freeRequest " << s -// << " : request:" << index -// << " now nullRequest:" -// << long(PstreamGlobals::outstandingRequests_(index)) << endl; -// } -// } -//} - - // ************************************************************************* //