diff --git a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C index fa0dd946bd53bc6d6878b4298b9806354cb3dfdc..b14da4cca16b89dabf07850b04741b4647664336 100644 --- a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C +++ b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C @@ -356,6 +356,9 @@ int main(int argc, char *argv[]) ) ); + // Give file handler a chance to determine the output directory + const_cast<fileOperation&>(fileHandler()).setNProcs(nDomains); + if (decomposeFieldsOnly) { // Sanity check on previously decomposed case @@ -395,22 +398,42 @@ int main(int argc, char *argv[]) Info<< "Removing " << nProcs << " existing processor directories" << endl; - fileHandler().rmDir + // Remove existing processors directory + fileNameList dirs ( - runTime.path()/word("processors"), - true // silent (may not have been collated) + fileHandler().readDir + ( + runTime.path(), + fileName::Type::DIRECTORY + ) ); - - // remove existing processor dirs - // reverse order to avoid gaps if someone interrupts the process - for (label proci = nProcs-1; proci >= 0; --proci) + forAllReverse(dirs, diri) { - fileName procDir - ( - runTime.path()/(word("processor") + name(proci)) - ); + const fileName& d = dirs[diri]; + + // Starts with 'processors' + if (d.find("processors") == 0) + { + if (fileHandler().exists(d)) + { + fileHandler().rmDir(d); + } + } - fileHandler().rmDir(procDir); + // Starts with 'processor' + if (d.find("processor") == 0) + { + // Check that integer after processor + fileName num(d.substr(9)); + label proci = -1; + if (Foam::read(num.c_str(), proci)) + { + if (fileHandler().exists(d)) + { + fileHandler().rmDir(d); + } + } + } } procDirsProblem = false; diff --git a/applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C b/applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C index c8ab33cfa24ad2ae0151680620fff9d549e644fb..d807c192231bea0acf5f9a14b014ae260b23a118 100644 --- a/applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C +++ b/applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C @@ -216,6 +216,9 @@ int main(int argc, char *argv[]) << exit(FatalError); } + // Warn fileHandler of number of processors + const_cast<fileOperation&>(fileHandler()).setNProcs(nProcs); + // Create the processor databases PtrList<Time> databases(nProcs); diff --git a/applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C b/applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C index 5da86f74521634301b5a315c7f5dce399070031a..3dc56dc5d0fe2e968cc69b2ca5893c6b8640566e 100644 --- a/applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C +++ b/applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C @@ -33,6 +33,7 @@ License #include "Time.H" #include "patchZones.H" #include "IOobjectList.H" +#include "collatedFileOperation.H" // VTK includes #include "vtkDataArraySelection.h" @@ -268,7 +269,7 @@ Foam::word Foam::vtkPVFoam::getReaderPartName(const int partId) const Foam::vtkPVFoam::vtkPVFoam ( - const char* const FileName, + const char* const vtkFileName, vtkPVFoamReader* reader ) : @@ -294,12 +295,19 @@ Foam::vtkPVFoam::vtkPVFoam { if (debug) { - Info<< "vtkPVFoam - " << FileName << nl; + Info<< "vtkPVFoam - " << vtkFileName << nl; printMemory(); } + fileName FileName(vtkFileName); + + // Make sure not to use the threaded version - it does not like + // being loaded as a shared library - static cleanup order is problematic. + // For now just disable the threaded writer. + fileOperations::collatedFileOperation::maxThreadFileBufferSize = 0; + // avoid argList and get rootPath/caseName directly from the file - fileName fullCasePath(fileName(FileName).path()); + fileName fullCasePath(FileName.path()); if (!isDir(fullCasePath)) { @@ -314,8 +322,20 @@ Foam::vtkPVFoam::vtkPVFoam setEnv("FOAM_EXECUTABLE", "paraview", false); // Set the case as an environment variable - some BCs might use this + if (fullCasePath.name().find("processors", 0) == 0) + { + // FileName e.g. "cavity/processors256/processor1.OpenFOAM + // Remove the processors section so it goes into processorDDD + // checking below. + fullCasePath = fullCasePath.path()/fileName(FileName.name()).lessExt(); + } + + if (fullCasePath.name().find("processor", 0) == 0) { + // Give filehandler opportunity to analyse number of processors + (void)fileHandler().filePath(fullCasePath); + const fileName globalCase = fullCasePath.path(); setEnv("FOAM_CASE", globalCase, true); diff --git a/src/OSspecific/POSIX/POSIX.C b/src/OSspecific/POSIX/POSIX.C index 5e1f85991218a2bc36d9bea23dc930a0213a0a14..061103e1cf753281777009b18414d037fbe414d4 100644 --- a/src/OSspecific/POSIX/POSIX.C +++ b/src/OSspecific/POSIX/POSIX.C @@ -489,6 +489,10 @@ mode_t Foam::mode(const fileName& name, const bool followLink) if (POSIX::debug) { Pout<< FUNCTION_NAME << " : name:" << name << endl; + if ((POSIX::debug & 2) && !Pstream::master()) + { + error::printStack(Pout); + } } // Ignore an empty name => always 0 @@ -516,10 +520,6 @@ Foam::fileName::Type Foam::type(const fileName& name, const bool followLink) if (POSIX::debug) { Pout<< FUNCTION_NAME << " : name:" << name << endl; - if ((POSIX::debug & 2) && !Pstream::master()) - { - error::printStack(Pout); - } } mode_t m = mode(name, followLink); @@ -1678,142 +1678,4 @@ Foam::fileNameList Foam::dlLoaded() } -static Foam::DynamicList<Foam::autoPtr<pthread_t>> threads_; -static Foam::DynamicList<Foam::autoPtr<pthread_mutex_t>> mutexes_; - -Foam::label Foam::allocateThread() -{ - forAll(threads_, i) - { - if (!threads_[i].valid()) - { - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : reusing index:" << i << endl; - } - // Reuse entry - threads_[i].reset(new pthread_t()); - return i; - } - } - - const label index = threads_.size(); - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : new index:" << index << endl; - } - threads_.append(autoPtr<pthread_t>(new pthread_t())); - - return index; -} - - -void Foam::createThread -( - const label index, - void *(*start_routine) (void *), - void *arg -) -{ - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : index:" << index << endl; - } - if (pthread_create(&threads_[index](), nullptr, start_routine, arg)) - { - FatalErrorInFunction - << "Failed starting thread " << index << exit(FatalError); - } -} - - -void Foam::joinThread(const label index) -{ - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : index:" << index << endl; - } - if (pthread_join(threads_[index](), nullptr)) - { - FatalErrorInFunction << "Failed joining thread " << index - << exit(FatalError); - } -} - - -void Foam::freeThread(const label index) -{ - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : index:" << index << endl; - } - threads_[index].clear(); -} - - -Foam::label Foam::allocateMutex() -{ - forAll(mutexes_, i) - { - if (!mutexes_[i].valid()) - { - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : reusing index:" << i << endl; - } - // Reuse entry - mutexes_[i].reset(new pthread_mutex_t()); - return i; - } - } - - const label index = mutexes_.size(); - - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : new index:" << index << endl; - } - mutexes_.append(autoPtr<pthread_mutex_t>(new pthread_mutex_t())); - return index; -} - - -void Foam::lockMutex(const label index) -{ - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : index:" << index << endl; - } - if (pthread_mutex_lock(&mutexes_[index]())) - { - FatalErrorInFunction << "Failed locking mutex " << index - << exit(FatalError); - } -} - - -void Foam::unlockMutex(const label index) -{ - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : index:" << index << endl; - } - if (pthread_mutex_unlock(&mutexes_[index]())) - { - FatalErrorInFunction << "Failed unlocking mutex " << index - << exit(FatalError); - } -} - - -void Foam::freeMutex(const label index) -{ - if (POSIX::debug) - { - Pout<< FUNCTION_NAME << " : index:" << index << endl; - } - mutexes_[index].clear(); -} - - // ************************************************************************* // diff --git a/src/OpenFOAM/Make/files b/src/OpenFOAM/Make/files index b0edac8799ca533f86583b1495d54b2e29e04ee8..427791f2c2b1c35004bf6fdda3ee9cac7fdc456b 100644 --- a/src/OpenFOAM/Make/files +++ b/src/OpenFOAM/Make/files @@ -11,9 +11,11 @@ global/etcFiles/etcFiles.C fileOps = global/fileOperations $(fileOps)/fileOperation/fileOperation.C +$(fileOps)/fileOperationInitialise/fileOperationInitialise.C $(fileOps)/uncollatedFileOperation/uncollatedFileOperation.C $(fileOps)/masterUncollatedFileOperation/masterUncollatedFileOperation.C $(fileOps)/collatedFileOperation/collatedFileOperation.C +$(fileOps)/collatedFileOperation/hostCollatedFileOperation.C $(fileOps)/collatedFileOperation/threadedCollatedOFstream.C $(fileOps)/collatedFileOperation/OFstreamCollator.C @@ -288,7 +290,6 @@ $(Time)/TimeIO.C $(Time)/findTimes.C $(Time)/subCycleTime.C $(Time)/subLoopTime.C -$(Time)/findInstance.C $(Time)/timeSelector.C $(Time)/instant/instant.C diff --git a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C index 7667f0012f49f8b1142c833d6f3b437ed317d93a..a99547c2e3a6cc6b6473d399dad428d4929d86a5 100644 --- a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C +++ b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -681,14 +681,15 @@ void Foam::decomposedBlockData::gather List<int> recvOffsets; List<int> recvSizes; - if (UPstream::master()) + if (UPstream::master(comm)) { recvOffsets.setSize(nProcs); forAll(recvOffsets, proci) { + // Note: truncating long int to int since UPstream::gather limited + // to ints recvOffsets[proci] = - reinterpret_cast<char*>(&datas[proci]) - - data0Ptr; + int(reinterpret_cast<char*>(&datas[proci]) - data0Ptr); } recvSizes.setSize(nProcs, sizeof(label)); } @@ -748,7 +749,8 @@ void Foam::decomposedBlockData::gatherSlaveData && (UPstream::myProcNo(comm) < startProc+nProcs) ) { - nSend = data.byteSize(); + // Note: UPstream::gather limited to int + nSend = int(data.byteSize()); } UPstream::gather @@ -764,6 +766,46 @@ void Foam::decomposedBlockData::gatherSlaveData } +Foam::label Foam::decomposedBlockData::calcNumProcs +( + const label comm, + const off_t maxBufferSize, + const labelUList& recvSizes, + const label startProci +) +{ + const label nProcs = UPstream::nProcs(comm); + + label nSendProcs = -1; + if (UPstream::master(comm)) + { + off_t totalSize = recvSizes[startProci]; + label proci = startProci+1; + while (proci < nProcs && (totalSize+recvSizes[proci] < maxBufferSize)) + { + totalSize += recvSizes[proci]; + proci++; + } + + nSendProcs = proci-startProci; + } + + // Scatter nSendProcs + label n; + UPstream::scatter + ( + reinterpret_cast<const char*>(&nSendProcs), + List<int>(nProcs, sizeof(nSendProcs)), + List<int>(nProcs, 0), + reinterpret_cast<char*>(&n), + sizeof(n), + comm + ); + + return n; +} + + bool Foam::decomposedBlockData::writeBlocks ( const label comm, @@ -772,8 +814,7 @@ bool Foam::decomposedBlockData::writeBlocks const UList<char>& data, const labelUList& recvSizes, - const bool haveSlaveData, - const List<char>& slaveData, + const PtrList<SubList<char>>& slaveData, const UPstream::commsTypes commsType, const bool syncReturnState @@ -784,17 +825,15 @@ bool Foam::decomposedBlockData::writeBlocks Pout<< "decomposedBlockData::writeBlocks:" << " stream:" << (osPtr.valid() ? osPtr().name() : "invalid") << " data:" << data.size() - << " haveSlaveData:" << haveSlaveData << " (master only) slaveData:" << slaveData.size() << " commsType:" << Pstream::commsTypeNames[commsType] << endl; } const label nProcs = UPstream::nProcs(comm); - bool ok = true; - if (haveSlaveData) + if (slaveData.size()) { // Already have gathered the slave data. communicator only used to // check who is the master @@ -821,8 +860,7 @@ bool Foam::decomposedBlockData::writeBlocks os << nl << nl << "// Processor" << proci << nl; start[proci] = os.stdStream().tellp(); - os << SubList<char>(slaveData, recvSizes[proci], slaveOffset); - + os << slaveData[proci]; slaveOffset += recvSizes[proci]; } @@ -897,44 +935,24 @@ bool Foam::decomposedBlockData::writeBlocks // maxMasterFileBufferSize // Starting slave processor and number of processors - labelPair startAndSize(1, nProcs-1); + label startProc = 1; + label nSendProcs = nProcs-1; - while (startAndSize[1] > 0) + while (nSendProcs > 0) { - labelPair masterData(startAndSize); - if (UPstream::master(comm)) - { - label totalSize = recvSizes[masterData[0]]; - label proci = masterData[0]+1; - while - ( - proci < nProcs - && ( - totalSize+recvSizes[proci] - < fileOperations::masterUncollatedFileOperation:: - maxMasterFileBufferSize - ) - ) - { - totalSize += recvSizes[proci]; - ++proci; - } - - masterData[1] = proci-masterData[0]; - } - - // Scatter masterData - UPstream::scatter + nSendProcs = calcNumProcs ( - reinterpret_cast<const char*>(masterData.cdata()), - List<int>(nProcs, sizeof(masterData)), - List<int>(nProcs, 0), - reinterpret_cast<char*>(startAndSize.data()), - sizeof(startAndSize), - comm + comm, + off_t + ( + fileOperations::masterUncollatedFileOperation:: + maxMasterFileBufferSize + ), + recvSizes, + startProc ); - if (startAndSize[0] == nProcs || startAndSize[1] == 0) + if (startProc == nProcs || nSendProcs == 0) { break; } @@ -949,8 +967,8 @@ bool Foam::decomposedBlockData::writeBlocks data, recvSizes, - startAndSize[0], // startProc, - startAndSize[1], // nProcs, + startProc, // startProc, + nSendProcs, // nProcs, sliceOffsets, recvData @@ -963,9 +981,9 @@ bool Foam::decomposedBlockData::writeBlocks // Write slaves for ( - label proci = startAndSize[0]; - proci < startAndSize[0]+startAndSize[1]; - ++proci + label proci = startProc; + proci < startProc+nSendProcs; + proci++ ) { os << nl << nl << "// Processor" << proci << nl; @@ -981,7 +999,7 @@ bool Foam::decomposedBlockData::writeBlocks } } - startAndSize[0] += startAndSize[1]; + startProc += nSendProcs; } if (UPstream::master(comm)) @@ -1027,7 +1045,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const ); IOobject io(*this); - if (Pstream::master()) + if (Pstream::master(comm_)) { IStringStream is ( @@ -1043,7 +1061,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const // version string versionString(os.version().str()); - Pstream::scatter(versionString); + Pstream::scatter(versionString, Pstream::msgType(), comm_); // stream string formatString; @@ -1051,21 +1069,21 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const OStringStream os; os << os.format(); formatString = os.str(); - Pstream::scatter(formatString); + Pstream::scatter(formatString, Pstream::msgType(), comm_); } //word masterName(name()); - //Pstream::scatter(masterName); + //Pstream::scatter(masterName, Pstream::msgType(), comm_); - Pstream::scatter(io.headerClassName()); - Pstream::scatter(io.note()); + Pstream::scatter(io.headerClassName(), Pstream::msgType(), comm_); + Pstream::scatter(io.note(), Pstream::msgType(), comm_); //Pstream::scatter(io.instance(), Pstream::msgType(), comm); //Pstream::scatter(io.local(), Pstream::msgType(), comm); fileName masterLocation(instance()/db().dbDir()/local()); - Pstream::scatter(masterLocation); + Pstream::scatter(masterLocation, Pstream::msgType(), comm_); - if (!Pstream::master()) + if (!Pstream::master(comm_)) { writeHeader ( @@ -1081,7 +1099,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const os.writeQuoted(str, false); - if (!Pstream::master()) + if (!Pstream::master(comm_)) { IOobject::writeEndDivider(os); } @@ -1108,10 +1126,10 @@ bool Foam::decomposedBlockData::writeObject } labelList recvSizes; - gather(comm_, this->byteSize(), recvSizes); + gather(comm_, label(this->byteSize()), recvSizes); List<std::streamoff> start; - List<char> slaveData; // dummy already received slave data + PtrList<SubList<char>> slaveData; // dummy slave data return writeBlocks ( comm_, @@ -1119,7 +1137,6 @@ bool Foam::decomposedBlockData::writeObject start, *this, recvSizes, - false, // don't have slave data slaveData, commsType_ ); diff --git a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H index 2223846d1b162f9c7fe7a434dc9b0783c31fe433..7eb95c0e89bd5d4f8f30976f1b84ab9800ce4615 100644 --- a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H +++ b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -66,6 +66,16 @@ protected: // Protected member functions + //- Helper: determine number of processors whose recvSizes fits + // ito maxBufferSize + static label calcNumProcs + ( + const label comm, + const off_t maxBufferSize, + const labelUList& recvSizes, + const label startProci + ); + //- Read data into *this. ISstream is only valid on master. static bool readBlocks ( @@ -202,12 +212,12 @@ public: const label comm, autoPtr<OSstream>& osPtr, List<std::streamoff>& start, - const UList<char>&, + const UList<char>& masterData, const labelUList& recvSizes, - const bool haveSlaveData, // does master have slaveData - const List<char>& slaveData, // optional slave data (on master) + // optional slave data (on master) + const PtrList<SubList<char>>& slaveData, const UPstream::commsTypes, const bool syncReturnState = true diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index ea72f84912cf0affdc75a2e3d6b4fea07556e982..6d43ae2c18ceb2524bd7c959b3d193e463bc619b 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -372,10 +372,8 @@ public: static void addValidParOptions(HashTable<string>& validParOptions); //- Initialisation function called from main - // Spawns slave processes and initialises inter-communication. - // \note warns if MPI has already been initialized. - // Fatal if MPI has already been finalized. - static bool init(int& argc, char**& argv); + // Spawns slave processes and initialises inter-communication + static bool init(int& argc, char**& argv, const bool needsThread); //- Special purpose initialisation function. // Performs a basic MPI_Init without any other setup. diff --git a/src/OpenFOAM/db/Time/Time.C b/src/OpenFOAM/db/Time/Time.C index f524dbad5c47ed33735c3e52658ea94bb239daf3..e0a6cae7c51839352827646647f044c29d0d248f 100644 --- a/src/OpenFOAM/db/Time/Time.C +++ b/src/OpenFOAM/db/Time/Time.C @@ -714,6 +714,36 @@ Foam::instantList Foam::Time::times() const } +Foam::word Foam::Time::findInstance +( + const fileName& dir, + const word& name, + const IOobject::readOption rOpt, + const word& stopInstance +) const +{ + IOobject startIO + ( + name, // name might be empty! + timeName(), + dir, + *this, + rOpt + ); + + IOobject io + ( + fileHandler().findInstance + ( + startIO, + timeOutputValue(), + stopInstance + ) + ); + return io.instance(); +} + + Foam::word Foam::Time::findInstancePath ( const fileName& directory, diff --git a/src/OpenFOAM/db/Time/Time.H b/src/OpenFOAM/db/Time/Time.H index 1deea1274926890aeae25940ac47c5f02ea171da..5ca64171409109a5a12a97f5bfc34169c89b45d2 100644 --- a/src/OpenFOAM/db/Time/Time.H +++ b/src/OpenFOAM/db/Time/Time.H @@ -196,9 +196,6 @@ protected: //- Read the control dictionary and set the write controls etc. virtual void readDict(); - //- Find IOobject in the objectPath - static bool exists(IOobject& io); - private: diff --git a/src/OpenFOAM/db/Time/findInstance.C b/src/OpenFOAM/db/Time/findInstance.C deleted file mode 100644 index f10fad0d384144cb77e808722673fc054da3cb7b..0000000000000000000000000000000000000000 --- a/src/OpenFOAM/db/Time/findInstance.C +++ /dev/null @@ -1,236 +0,0 @@ -/*---------------------------------------------------------------------------*\ - ========= | - \\ / F ield | OpenFOAM: The Open Source CFD Toolbox - \\ / O peration | - \\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation - \\/ M anipulation | -------------------------------------------------------------------------------- -License - This file is part of OpenFOAM. - - OpenFOAM is free software: you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - OpenFOAM is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. - -Description - If "name" is empty: return the location of "directory" - If "name" is not empty: return the location of "directory" containing the - file "name". - Used in reading mesh data. - -\*---------------------------------------------------------------------------*/ - -#include "Time.H" -#include "IOobject.H" -#include "IOList.H" - -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - -bool Foam::Time::exists(IOobject& io) -{ - // Generate filename for object - fileName objPath(fileHandler().objectPath(io, word::null)); - - // Test for either directory or a (valid) file & IOobject - bool ok; - if (io.name().empty()) - { - ok = fileHandler().isDir(objPath); - } - else - { - ok = - fileHandler().isFile(objPath) - && io.typeHeaderOk<IOList<label>>(false);// object with local scope - } - - if (!ok) - { - // Re-test with raw objectPath. This is for backwards - // compatibility - fileName originalPath(io.objectPath()); - if (originalPath != objPath) - { - // Test for either directory or a (valid) file & IOobject - if (io.name().empty()) - { - ok = fileHandler().isDir(originalPath); - } - else - { - ok = - fileHandler().isFile(originalPath) - && io.typeHeaderOk<IOList<label>>(false); - } - } - } - - return ok; -} - - -Foam::word Foam::Time::findInstance -( - const fileName& dir, - const word& name, - const IOobject::readOption rOpt, - const word& stopInstance -) const -{ - // Note: - if name is empty, just check the directory itself - // - check both for isFile and headerOk since the latter does a - // filePath so searches for the file. - // - check for an object with local file scope (so no looking up in - // parent directory in case of parallel) - - { - IOobject io - ( - name, // name might be empty! - timeName(), - dir, - *this - ); - - if (exists(io)) - { - if (debug) - { - InfoInFunction - << "Found exact match for \"" << name - << "\" in " << timeName()/dir - << endl; - } - - return timeName(); - } - } - - // Search back through the time directories to find the time - // closest to and lower than current time - - instantList ts = times(); - label instanceI; - - for (instanceI = ts.size()-1; instanceI >= 0; --instanceI) - { - if (ts[instanceI].value() <= timeOutputValue()) - { - break; - } - } - - // continue searching from here - for (; instanceI >= 0; --instanceI) - { - IOobject io - ( - name, // name might be empty! - ts[instanceI].name(), - dir, - *this - ); - - if (exists(io)) - { - if (debug) - { - InfoInFunction - << "Found instance match for \"" << name - << "\" in " << ts[instanceI].name()/dir - << endl; - } - - return ts[instanceI].name(); - } - - // Check if hit minimum instance - if (ts[instanceI].name() == stopInstance) - { - if (debug) - { - //InfoInFunction - Pout<< "findInstance : " - << "Hit stopInstance " << stopInstance - << endl; - } - - if - ( - rOpt == IOobject::MUST_READ - || rOpt == IOobject::MUST_READ_IF_MODIFIED - ) - { - if (name.empty()) - { - FatalErrorInFunction - << "Cannot find directory " - << dir << " in times " << timeName() - << " down to " << stopInstance - << exit(FatalError); - } - else - { - FatalErrorInFunction - << "Cannot find file \"" << name << "\" in directory " - << dir << " in times " << timeName() - << " down to " << stopInstance - << exit(FatalError); - } - } - - return ts[instanceI].name(); - } - } - - - // not in any of the time directories, try constant - - // Note. This needs to be a hard-coded constant, rather than the - // constant function of the time, because the latter points to - // the case constant directory in parallel cases - - IOobject io - ( - name, - constant(), - dir, - *this - ); - - if (exists(io)) - { - if (debug) - { - InfoInFunction - << "Found constant match for \"" << name - << "\" in " << constant()/dir - << endl; - } - - return constant(); - } - - if (rOpt == IOobject::MUST_READ || rOpt == IOobject::MUST_READ_IF_MODIFIED) - { - FatalErrorInFunction - << "Cannot find file \"" << name << "\" in directory " - << dir << " in times " << timeName() - << " down to " << constant() - << exit(FatalError); - } - - return constant(); -} - - -// ************************************************************************* // diff --git a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C index bdabea535250234e7abee730999c06e6e936c55c..48d1948d33b1e514b187998d54d78614a88f26fe 100644 --- a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C +++ b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation \\/ M anipulation | Copyright (C) 2015-2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License @@ -183,7 +183,8 @@ void Foam::regIOobject::close() if (IFstream::debug) { Pout<< "regIOobject::close() : " - << "finished reading " << isPtr_().name() + << "finished reading " + << (isPtr_.valid() ? isPtr_().name() : "dummy") << endl; } diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C index 21a825c6cdf647721ad857d24c8733574aacfd48..d62cabed2f4c470bf27a5d336c0c62cc0bc32cd9 100644 --- a/src/OpenFOAM/global/argList/argList.C +++ b/src/OpenFOAM/global/argList/argList.C @@ -40,8 +40,9 @@ License #include "foamVersion.H" #include "stringOps.H" #include "CStringList.H" -#include "uncollatedFileOperation.H" -#include "masterUncollatedFileOperation.H" +#include "stringListOps.H" +#include "fileOperation.H" +#include "fileOperationInitialise.H" #include "IOmanip.H" #include <cctype> @@ -83,6 +84,12 @@ Foam::argList::initValidTables::initValidTables() "decomposeParDict", "file", "read decomposePar dictionary from specified location" ); + argList::addOption + ( + "hostRoots", "(((host1 dir1) .. (hostN dirN))", + "slave root directories (per host) for distributed running" + ); + validParOptions.set("hostRoots", "((host1 dir1) .. (hostN dirN))"); argList::addBoolOption ( @@ -324,6 +331,7 @@ void Foam::argList::noParallel() removeOption("parallel"); removeOption("roots"); removeOption("decomposeParDict"); + removeOption("hostRoots"); validParOptions.clear(); } @@ -687,6 +695,34 @@ Foam::argList::argList options_(argc), distributed_(false) { + // Check for fileHandler + word handlerType(getEnv("FOAM_FILEHANDLER")); + for (int argI = 0; argI < argc; ++argI) + { + if (argv[argI][0] == '-') + { + const char *optionName = &argv[argI][1]; + if (string(optionName) == "fileHandler") + { + handlerType = argv[argI+1]; + break; + } + } + } + if (handlerType.empty()) + { + handlerType = fileOperation::defaultFileHandler; + } + + // Detect any parallel options + bool needsThread = fileOperations::fileOperationInitialise::New + ( + handlerType, + argc, + argv + )().needsThreading(); + + // Check if this run is a parallel run by searching for any parallel option // If found call runPar which might filter argv for (int argi = 1; argi < argc; ++argi) @@ -697,7 +733,7 @@ Foam::argList::argList if (validParOptions.found(optName)) { - parRunControl_.runPar(argc, argv); + parRunControl_.runPar(argc, argv, needsThread); break; } } @@ -942,6 +978,58 @@ void Foam::argList::parse Foam::fileHandler(handler); } + + stringList slaveProcs; + stringList slaveMachine; + const int writeHostsSwitch = debug::infoSwitch("writeHosts", 1); + + // Collect slave machine/pid, and check that the build is identical + if (parRunControl_.parRun()) + { + if (Pstream::master()) + { + slaveProcs.setSize(Pstream::nProcs() - 1); + slaveMachine.setSize(Pstream::nProcs() - 1); + label proci = 0; + for + ( + int slave = Pstream::firstSlave(); + slave <= Pstream::lastSlave(); + slave++ + ) + { + IPstream fromSlave(Pstream::commsTypes::scheduled, slave); + + string slaveBuild; + label slavePid; + fromSlave >> slaveBuild >> slaveMachine[proci] >> slavePid; + slaveProcs[proci] = slaveMachine[proci] + "." + name(slavePid); + proci++; + + // Check build string to make sure all processors are running + // the same build + if (slaveBuild != Foam::FOAMbuild) + { + FatalErrorIn(executable()) + << "Master is running version " << Foam::FOAMbuild + << "; slave " << proci << " is running version " + << slaveBuild + << exit(FatalError); + } + } + } + else + { + OPstream toMaster + ( + Pstream::commsTypes::scheduled, + Pstream::masterNo() + ); + toMaster << string(Foam::FOAMbuild) << hostName() << pid(); + } + } + + // Case is a single processor run unless it is running parallel int nProcs = 1; @@ -999,6 +1087,52 @@ void Foam::argList::parse dictNProcs = roots.size()+1; } } + else if (options_.found("hostRoots")) + { + source = "-hostRoots"; + IStringStream is(options_["hostRoots"]); + List<Tuple2<wordRe, fileName>> hostRoots(is); + + roots.setSize(Pstream::nProcs()-1); + forAll(hostRoots, i) + { + const Tuple2<wordRe, fileName>& hostRoot = hostRoots[i]; + const wordRe& re = hostRoot.first(); + labelList matchedRoots(findStrings(re, slaveMachine)); + forAll(matchedRoots, matchi) + { + label slavei = matchedRoots[matchi]; + if (roots[slavei] != wordRe()) + { + FatalErrorInFunction + << "Slave " << slaveMachine[slavei] + << " has multiple matching roots in " + << hostRoots << exit(FatalError); + } + else + { + roots[slavei] = hostRoot.second(); + } + } + } + + // Check + forAll(roots, slavei) + { + if (roots[slavei] == wordRe()) + { + FatalErrorInFunction + << "Slave " << slaveMachine[slavei] + << " has no matching roots in " + << hostRoots << exit(FatalError); + } + } + + if (roots.size() != 1) + { + dictNProcs = roots.size()+1; + } + } else if (checkProcessorDirectories_) { // Use values from decomposeParDict, the location was already @@ -1172,55 +1306,6 @@ void Foam::argList::parse case_ = globalCase_; } - stringList slaveProcs; - const int writeHostsSwitch = debug::infoSwitch("writeHosts", 1); - - // Collect slave machine/pid, and check that the build is identical - if (parRunControl_.parRun()) - { - if (Pstream::master()) - { - slaveProcs.setSize(Pstream::nProcs() - 1); - label proci = 0; - for - ( - int slave = Pstream::firstSlave(); - slave <= Pstream::lastSlave(); - slave++ - ) - { - IPstream fromSlave(Pstream::commsTypes::scheduled, slave); - - string slaveBuild; - string slaveMachine; - label slavePid; - fromSlave >> slaveBuild >> slaveMachine >> slavePid; - - slaveProcs[proci++] = slaveMachine + "." + name(slavePid); - - // Check build string to make sure all processors are running - // the same build - if (slaveBuild != Foam::FOAMbuild) - { - FatalErrorIn(executable()) - << "Master is running version " << Foam::FOAMbuild - << "; slave " << proci << " is running version " - << slaveBuild - << exit(FatalError); - } - } - } - else - { - OPstream toMaster - ( - Pstream::commsTypes::scheduled, - Pstream::masterNo() - ); - toMaster << string(Foam::FOAMbuild) << hostName() << pid(); - } - } - // Keep or discard slave and root information for reporting: if (Pstream::master() && parRunControl_.parRun()) { @@ -1412,6 +1497,7 @@ bool Foam::argList::unsetOption(const word& optName) optName == "case" || optName == "parallel" || optName == "roots" + || optName == "hostRoots" ) { FatalErrorInFunction diff --git a/src/OpenFOAM/global/argList/parRun.H b/src/OpenFOAM/global/argList/parRun.H index 6fa5f1e7d6a1c09507fdd480fbb9ee775d1dc7ca..3c21a6ad0c4822464d698ec50b085dc27a72d68a 100644 --- a/src/OpenFOAM/global/argList/parRun.H +++ b/src/OpenFOAM/global/argList/parRun.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -70,11 +70,11 @@ public: } //- Initialize Pstream for a parallel run - void runPar(int& argc, char**& argv) + void runPar(int& argc, char**& argv, const bool needsThread) { RunPar = true; - if (!Pstream::init(argc, argv)) + if (!Pstream::init(argc, argv, needsThread)) { Info<< "Failed to start parallel run" << endl; Pstream::exit(1); diff --git a/src/OpenFOAM/global/debug/debug.C b/src/OpenFOAM/global/debug/debug.C index bf5b90582ed987a4156b1db9f1f083ef6124d866..2b7206cc4fee8f5d93ab9d9161ba8c8ca4cb3f4e 100644 --- a/src/OpenFOAM/global/debug/debug.C +++ b/src/OpenFOAM/global/debug/debug.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -95,21 +95,31 @@ Foam::dictionary& Foam::debug::controlDict() { if (!controlDictPtr_) { - fileNameList controlDictFiles = findEtcFiles("controlDict", true); - controlDictPtr_ = new dictionary(); - forAllReverse(controlDictFiles, cdfi) + string controlDictString(getEnv("FOAM_CONTROLDICT")); + if (!controlDictString.empty()) { - IFstream ifs(controlDictFiles[cdfi]); - - if (!ifs.good()) + // Read from environment + IStringStream is(controlDictString); + controlDictPtr_ = new dictionary(is); + } + else + { + fileNameList controlDictFiles = findEtcFiles("controlDict", true); + controlDictPtr_ = new dictionary(); + forAllReverse(controlDictFiles, cdfi) { - SafeFatalIOErrorInFunction - ( - ifs, - "Cannot open controlDict" - ); + IFstream ifs(controlDictFiles[cdfi]); + + if (!ifs.good()) + { + SafeFatalIOErrorInFunction + ( + ifs, + "Cannot open controlDict" + ); + } + controlDictPtr_->merge(dictionary(ifs)); } - controlDictPtr_->merge(dictionary(ifs)); } } diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C index d6d1e73036789cbcea80abb310a65df07626e2a9..616846413134533dcfab185ff0aaa7eeaf542526 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -26,6 +26,7 @@ License #include "OFstreamCollator.H" #include "OFstream.H" #include "decomposedBlockData.H" +#include "masterUncollatedFileOperation.H" // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // @@ -44,8 +45,7 @@ bool Foam::OFstreamCollator::writeFile const fileName& fName, const string& masterData, const labelUList& recvSizes, - const bool haveSlaveData, // does master have slaveData - const UList<char>& slaveData, // on master: slave data + const PtrList<SubList<char>>& slaveData, // optional slave data IOstream::streamFormat fmt, IOstream::versionNumber ver, IOstream::compressionType cmp, @@ -54,9 +54,22 @@ bool Foam::OFstreamCollator::writeFile { if (debug) { - Pout<< "OFstreamCollator : Writing " << masterData.size() + Pout<< "OFstreamCollator : Writing master " << masterData.size() << " bytes to " << fName << " using comm " << comm << endl; + if (slaveData.size()) + { + Pout<< "OFstreamCollator : Slave data" << endl; + forAll(slaveData, proci) + { + if (slaveData.set(proci)) + { + Pout<< " " << proci + << " size:" << slaveData[proci].size() + << endl; + } + } + } } autoPtr<OSstream> osPtr; @@ -76,17 +89,20 @@ bool Foam::OFstreamCollator::writeFile ); // We don't have IOobject so cannot use IOobject::writeHeader - OSstream& os = osPtr(); - decomposedBlockData::writeHeader - ( - os, - ver, - fmt, - typeName, - "", - fName, - fName.name() - ); + if (!append) + { + OSstream& os = osPtr(); + decomposedBlockData::writeHeader + ( + os, + ver, + fmt, + typeName, + "", + fName, + fName.name() + ); + } } @@ -109,9 +125,13 @@ bool Foam::OFstreamCollator::writeFile start, slice, recvSizes, - haveSlaveData, slaveData, - UPstream::commsTypes::nonBlocking, //scheduled, + ( + fileOperations::masterUncollatedFileOperation:: + maxMasterFileBufferSize == 0 + ? UPstream::commsTypes::scheduled + : UPstream::commsTypes::nonBlocking + ), false // do not reduce return state ); @@ -132,7 +152,11 @@ bool Foam::OFstreamCollator::writeFile { sum += recvSizes[i]; } - Pout<< " (overall " << sum << ")"; + // Use ostringstream to display long int (until writing these is + // supported) + std::ostringstream os; + os << sum; + Pout<< " (overall " << os.str() << ")"; } Pout<< " to " << fName << " using comm " << comm << endl; @@ -151,12 +175,13 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg) { writeData* ptr = nullptr; - lockMutex(handler.mutex_); - if (handler.objects_.size()) { - ptr = handler.objects_.pop(); + std::lock_guard<std::mutex> guard(handler.mutex_); + if (handler.objects_.size()) + { + ptr = handler.objects_.pop(); + } } - unlockMutex(handler.mutex_); if (!ptr) { @@ -164,6 +189,28 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg) } else { + // Convert storage to pointers + PtrList<SubList<char>> slaveData; + if (ptr->slaveData_.size()) + { + slaveData.setSize(ptr->slaveData_.size()); + forAll(slaveData, proci) + { + if (ptr->slaveData_.set(proci)) + { + slaveData.set + ( + proci, + new SubList<char> + ( + ptr->slaveData_[proci], + ptr->sizes_[proci] + ) + ); + } + } + } + bool ok = writeFile ( ptr->comm_, @@ -171,9 +218,7 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg) ptr->pathName_, ptr->data_, ptr->sizes_, - ptr->haveSlaveData_, - ptr->slaveData_, - + slaveData, ptr->format_, ptr->version_, ptr->compression_, @@ -196,9 +241,10 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg) Pout<< "OFstreamCollator : Exiting write thread " << endl; } - lockMutex(handler.mutex_); - handler.threadRunning_ = false; - unlockMutex(handler.mutex_); + { + std::lock_guard<std::mutex> guard(handler.mutex_); + handler.threadRunning_ = false; + } return nullptr; } @@ -211,12 +257,13 @@ void Foam::OFstreamCollator::waitForBufferSpace(const off_t wantedSize) const // Count files to be written off_t totalSize = 0; - lockMutex(mutex_); - forAllConstIter(FIFOStack<writeData*>, objects_, iter) { - totalSize += iter()->size(); + std::lock_guard<std::mutex> guard(mutex_); + forAllConstIter(FIFOStack<writeData*>, objects_, iter) + { + totalSize += iter()->size(); + } } - unlockMutex(mutex_); if (totalSize == 0 || (totalSize+wantedSize) <= maxBufferSize_) { @@ -225,13 +272,12 @@ void Foam::OFstreamCollator::waitForBufferSpace(const off_t wantedSize) const if (debug) { - lockMutex(mutex_); + std::lock_guard<std::mutex> guard(mutex_); Pout<< "OFstreamCollator : Waiting for buffer space." << " Currently in use:" << totalSize << " limit:" << maxBufferSize_ << " files:" << objects_.size() << endl; - unlockMutex(mutex_); } sleep(5); @@ -244,25 +290,34 @@ void Foam::OFstreamCollator::waitForBufferSpace(const off_t wantedSize) const Foam::OFstreamCollator::OFstreamCollator(const off_t maxBufferSize) : maxBufferSize_(maxBufferSize), - mutex_ - ( - maxBufferSize_ > 0 - ? allocateMutex() - : -1 - ), - thread_ + threadRunning_(false), + localComm_(UPstream::worldComm), + threadComm_ ( - maxBufferSize_ > 0 - ? allocateThread() - : -1 - ), + UPstream::allocateCommunicator + ( + localComm_, + identity(UPstream::nProcs(localComm_)) + ) + ) +{} + + +Foam::OFstreamCollator::OFstreamCollator +( + const off_t maxBufferSize, + const label comm +) +: + maxBufferSize_(maxBufferSize), threadRunning_(false), - comm_ + localComm_(comm), + threadComm_ ( UPstream::allocateCommunicator ( - UPstream::worldComm, - identity(UPstream::nProcs(UPstream::worldComm)) + localComm_, + identity(UPstream::nProcs(localComm_)) ) ) {} @@ -272,26 +327,19 @@ Foam::OFstreamCollator::OFstreamCollator(const off_t maxBufferSize) Foam::OFstreamCollator::~OFstreamCollator() { - if (threadRunning_) + if (thread_.valid()) { if (debug) { Pout<< "~OFstreamCollator : Waiting for write thread" << endl; } - - joinThread(thread_); - } - if (thread_ != -1) - { - freeThread(thread_); + thread_().join(); + thread_.clear(); } - if (mutex_ != -1) - { - freeMutex(mutex_); - } - if (comm_ != -1) + + if (threadComm_ != -1) { - UPstream::freeCommunicator(comm_); + UPstream::freeCommunicator(threadComm_); } } @@ -312,7 +360,8 @@ bool Foam::OFstreamCollator::write // Determine (on master) sizes to receive. Note: do NOT use thread // communicator labelList recvSizes; - decomposedBlockData::gather(Pstream::worldComm, data.size(), recvSizes); + decomposedBlockData::gather(localComm_, label(data.size()), recvSizes); + off_t totalSize = 0; label maxLocalSize = 0; { @@ -321,8 +370,8 @@ bool Foam::OFstreamCollator::write totalSize += recvSizes[proci]; maxLocalSize = max(maxLocalSize, recvSizes[proci]); } - Pstream::scatter(totalSize, Pstream::msgType(), Pstream::worldComm); - Pstream::scatter(maxLocalSize, Pstream::msgType(), Pstream::worldComm); + Pstream::scatter(totalSize, Pstream::msgType(), localComm_); + Pstream::scatter(maxLocalSize, Pstream::msgType(), localComm_); } if (maxBufferSize_ == 0 || maxLocalSize > maxBufferSize_) @@ -330,18 +379,17 @@ bool Foam::OFstreamCollator::write if (debug) { Pout<< "OFstreamCollator : non-thread gather and write of " << fName - << " using worldComm" << endl; + << " using local comm " << localComm_ << endl; } // Direct collating and writing (so master blocks until all written!) - const List<char> dummySlaveData; + const PtrList<SubList<char>> dummySlaveData; return writeFile ( - UPstream::worldComm, + localComm_, typeName, fName, data, recvSizes, - false, // no slave data provided yet dummySlaveData, fmt, ver, @@ -360,22 +408,28 @@ bool Foam::OFstreamCollator::write << fName << endl; } - if (Pstream::master()) + if (Pstream::master(localComm_)) { waitForBufferSpace(totalSize); } - // Allocate local buffer for all collated data + + // Receive in chunks of labelMax (2^31-1) since this is the maximum + // size that a List can be + autoPtr<writeData> fileAndDataPtr ( new writeData ( - comm_, // Note: comm not actually used anymore + threadComm_, // Note: comm not actually used anymore typeName, fName, - data, + ( + Pstream::master(localComm_) + ? data // Only used on master + : string::null + ), recvSizes, - true, // have slave data (collected below) fmt, ver, cmp, @@ -384,40 +438,84 @@ bool Foam::OFstreamCollator::write ); writeData& fileAndData = fileAndDataPtr(); - // Gather the slave data and insert into fileAndData - UList<char> slice(const_cast<char*>(data.data()), label(data.size())); - List<int> slaveOffsets; - decomposedBlockData::gatherSlaveData - ( - Pstream::worldComm, // Note: using simulation thread - slice, - recvSizes, + PtrList<List<char>>& slaveData = fileAndData.slaveData_; - 1, // startProc, - Pstream::nProcs()-1, // n procs + UList<char> slice(const_cast<char*>(data.data()), label(data.size())); - slaveOffsets, - fileAndData.slaveData_ - ); + slaveData.setSize(recvSizes.size()); - // Append to thread buffer - lockMutex(mutex_); - objects_.push(fileAndDataPtr.ptr()); - unlockMutex(mutex_); + // Gather all data onto master. Is done in local communicator since + // not in write thread. Note that we do not store in contiguous + // buffer since that would limit to 2G chars. + label startOfRequests = Pstream::nRequests(); + if (Pstream::master(localComm_)) + { + for (label proci = 1; proci < slaveData.size(); proci++) + { + slaveData.set(proci, new List<char>(recvSizes[proci])); + UIPstream::read + ( + UPstream::commsTypes::nonBlocking, + proci, + reinterpret_cast<char*>(slaveData[proci].begin()), + slaveData[proci].byteSize(), + Pstream::msgType(), + localComm_ + ); + } + } + else + { + if + ( + !UOPstream::write + ( + UPstream::commsTypes::nonBlocking, + 0, + reinterpret_cast<const char*>(slice.begin()), + slice.byteSize(), + Pstream::msgType(), + localComm_ + ) + ) + { + FatalErrorInFunction + << "Cannot send outgoing message. " + << "to:" << 0 << " nBytes:" + << label(slice.byteSize()) + << Foam::abort(FatalError); + } + } + Pstream::waitRequests(startOfRequests); - // Start thread if not running - lockMutex(mutex_); - if (!threadRunning_) { - createThread(thread_, writeAll, this); - if (debug) + std::lock_guard<std::mutex> guard(mutex_); + + // Append to thread buffer + objects_.push(fileAndDataPtr.ptr()); + + // Start thread if not running + if (!threadRunning_) { - Pout<< "OFstreamCollator : Started write thread " - << thread_ << endl; + if (thread_.valid()) + { + if (debug) + { + Pout<< "OFstreamCollator : Waiting for write thread" + << endl; + } + thread_().join(); + } + + if (debug) + { + Pout<< "OFstreamCollator : Starting write thread" + << endl; + } + thread_.reset(new std::thread(writeAll, this)); + threadRunning_ = true; } - threadRunning_ = true; } - unlockMutex(mutex_); return true; } @@ -426,57 +524,65 @@ bool Foam::OFstreamCollator::write if (debug) { Pout<< "OFstreamCollator : thread gather and write of " << fName - << " in thread " << thread_ - << " using communicator " << comm_ << endl; + << " using communicator " << threadComm_ << endl; } if (!UPstream::haveThreads()) { FatalErrorInFunction << "mpi does not seem to have thread support." - << "Please increase the buffer size 'maxThreadFileBufferSize'" + << " Make sure to set buffer size 'maxThreadFileBufferSize'" << " to at least " << totalSize << " to be able to do the collating before threading." << exit(FatalError); } - if (Pstream::master()) + if (Pstream::master(localComm_)) { waitForBufferSpace(data.size()); } - lockMutex(mutex_); - // Push all file info on buffer. Note that no slave data provided - // so it will trigger communication inside the thread - objects_.push - ( - new writeData + { + std::lock_guard<std::mutex> guard(mutex_); + + // Push all file info on buffer. Note that no slave data provided + // so it will trigger communication inside the thread + objects_.push ( - comm_, - typeName, - fName, - data, - recvSizes, - false, // Have no slave data; collect in thread - fmt, - ver, - cmp, - append - ) - ); - unlockMutex(mutex_); + new writeData + ( + threadComm_, + typeName, + fName, + data, + recvSizes, + fmt, + ver, + cmp, + append + ) + ); - lockMutex(mutex_); - if (!threadRunning_) - { - createThread(thread_, writeAll, this); - if (debug) + if (!threadRunning_) { - Pout<< "OFstreamCollator : Started write thread " << endl; + if (thread_.valid()) + { + if (debug) + { + Pout<< "OFstreamCollator : Waiting for write thread" + << endl; + } + thread_().join(); + } + + if (debug) + { + Pout<< "OFstreamCollator : Starting write thread" << endl; + } + thread_.reset(new std::thread(writeAll, this)); + threadRunning_ = true; } - threadRunning_ = true; } - unlockMutex(mutex_); return true; } diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H index bf4eeb571352dace2261354361c72b773d5f8d40..92ef9068bfb5355a2e64d6ab81b58334d1ddc760 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -51,9 +51,12 @@ SourceFiles #ifndef OFstreamCollator_H #define OFstreamCollator_H +#include <thread> +#include <mutex> #include "IOstream.H" #include "labelList.H" #include "FIFOStack.H" +#include "SubList.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -77,10 +80,7 @@ class OFstreamCollator const fileName pathName_; const string data_; const labelList sizes_; - - const bool haveSlaveData_; - List<char> slaveData_; - + PtrList<List<char>> slaveData_; const IOstream::streamFormat format_; const IOstream::versionNumber version_; const IOstream::compressionType compression_; @@ -93,7 +93,6 @@ class OFstreamCollator const fileName& pathName, const string& data, const labelList& sizes, - const bool haveSlaveData, IOstream::streamFormat format, IOstream::versionNumber version, IOstream::compressionType compression, @@ -105,7 +104,6 @@ class OFstreamCollator pathName_(pathName), data_(data), sizes_(sizes), - haveSlaveData_(haveSlaveData), slaveData_(0), format_(format), version_(version), @@ -116,27 +114,39 @@ class OFstreamCollator //- (approximate) size of master + any optional slave data off_t size() const { - return data_.size() + slaveData_.size(); + off_t sz = data_.size(); + forAll(slaveData_, i) + { + if (slaveData_.set(i)) + { + sz += slaveData_[i].size(); + } + } + return sz; } }; // Private data + //- Total amount of storage to use for object stack below const off_t maxBufferSize_; - //pthread_mutex_t mutex_; - label mutex_; + mutable std::mutex mutex_; - //pthread_t thread_; - label thread_; + autoPtr<std::thread> thread_; + //- Stack of files to write + contents FIFOStack<writeData*> objects_; + //- Whether thread is running (and not exited) bool threadRunning_; - //- Communicator to use for all parallel ops - label comm_; + //- Communicator to use for all parallel ops (in simulation thread) + label localComm_; + + //- Communicator to use for all parallel ops (in write thread) + label threadComm_; // Private Member Functions @@ -149,8 +159,7 @@ class OFstreamCollator const fileName& fName, const string& masterData, const labelUList& recvSizes, - const bool haveSlaveData, // (does master) have slave data - const UList<char>& slaveData, // (on master) all slave data + const PtrList<SubList<char>>& slaveData, IOstream::streamFormat fmt, IOstream::versionNumber ver, IOstream::compressionType cmp, @@ -176,6 +185,10 @@ public: //- Construct from buffer size. 0 = do not use thread OFstreamCollator(const off_t maxBufferSize); + //- Construct from buffer size (0 = do not use thread) and local + // thread + OFstreamCollator(const off_t maxBufferSize, const label comm); + //- Destructor virtual ~OFstreamCollator(); diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C index 37b8103c55aa0aa9ac7a39f417fc0ed691823970..ce680b6bd68b0ec750efe51b41219bc5e2e32076 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -57,12 +57,63 @@ namespace fileOperations float, collatedFileOperation::maxThreadFileBufferSize ); + + // Mark as needing threaded mpi + addNamedToRunTimeSelectionTable + ( + fileOperationInitialise, + collatedFileOperationInitialise, + word, + collated + ); } } // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // +Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks() +{ + labelList ioRanks; + + string ioRanksString(getEnv("FOAM_IORANKS")); + if (!ioRanksString.empty()) + { + IStringStream is(ioRanksString); + is >> ioRanks; + } + + return ioRanks; +} + + +bool Foam::fileOperations::collatedFileOperation::isMasterRank +( + const label proci +) +const +{ + if (Pstream::parRun()) + { + return Pstream::master(comm_); + } + else + { + // Use any IO ranks + if (ioRanks_.size()) + { + // Find myself in IO rank + return findIndex(ioRanks_, proci) != -1; + } + else + { + // Assume all in single communicator + return proci == 0; + } + } +} + + bool Foam::fileOperations::collatedFileOperation::appendObject ( const regIOobject& io, @@ -74,14 +125,12 @@ bool Foam::fileOperations::collatedFileOperation::appendObject { // Append to processors/ file - fileName prefix; - fileName postfix; - label proci = splitProcessorPath(io.objectPath(), prefix, postfix); + label proci = detectProcessorPath(io.objectPath()); if (debug) { - Pout<< "writeObject:" << " : For local object : " - << io.name() + Pout<< "collatedFileOperation::writeObject :" + << " For local object : " << io.name() << " appending processor " << proci << " data to " << pathName << endl; } @@ -93,12 +142,35 @@ bool Foam::fileOperations::collatedFileOperation::appendObject << exit(FatalError); } + const bool isMaster = isMasterRank(proci); + + // Determine the local rank if the pathName is a per-rank one + label localProci = proci; + { + fileName path, procDir, local; + label groupStart, groupSize, nProcs; + splitProcessorPath + ( + pathName, + path, + procDir, + local, + groupStart, + groupSize, + nProcs + ); + if (groupSize > 0 && groupStart != -1) + { + localProci = proci-groupStart; + } + } + // Create string from all data to write string buf; { OStringStream os(fmt, ver); - if (proci == 0) + if (isMaster) { if (!io.writeHeader(os)) { @@ -112,7 +184,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject return false; } - if (proci == 0) + if (isMaster) { IOobject::writeEndDivider(os); } @@ -121,8 +193,6 @@ bool Foam::fileOperations::collatedFileOperation::appendObject } - bool append = (proci > 0); - // Note: cannot do append + compression. This is a limitation // of ogzstream (or rather most compressed formats) @@ -132,7 +202,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject IOstream::BINARY, ver, IOstream::UNCOMPRESSED, // no compression - append + !isMaster ); if (!os.good()) @@ -142,7 +212,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject << exit(FatalIOError); } - if (proci == 0) + if (isMaster) { IOobject::writeBanner(os) << "FoamFile\n{\n" @@ -162,7 +232,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject const_cast<char*>(buf.data()), label(buf.size()) ); - os << nl << "// Processor" << proci << nl << slice << nl; + os << nl << "// Processor" << localProci << nl << slice << nl; return os.good(); } @@ -175,8 +245,23 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation const bool verbose ) : - masterUncollatedFileOperation(false), - writer_(maxThreadFileBufferSize) + masterUncollatedFileOperation + ( + ( + ioRanks().size() + ? UPstream::allocateCommunicator + ( + UPstream::worldComm, + subRanks(Pstream::nProcs()) + ) + : UPstream::worldComm + ), + false + ), + myComm_(comm_), + writer_(maxThreadFileBufferSize, comm_), + nProcs_(Pstream::nProcs()), + ioRanks_(ioRanks()) { if (verbose) { @@ -195,15 +280,37 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation { Info<< " Threading activated " "since maxThreadFileBufferSize > 0." << nl - << " Requires thread support enabled in MPI, " - "otherwise the simulation" << nl - << " may \"hang\". If thread support cannot be " - "enabled, deactivate threading" << nl - << " by setting maxThreadFileBufferSize to 0 in " - "the OpenFOAM etc/controlDict" + << " Requires large enough buffer to collect all data" + " or thread support " << nl + << " enabled in MPI. If thread support cannot be " + "enabled, deactivate" << nl + << " threading by setting maxThreadFileBufferSize " + "to 0 in" << nl + << " $FOAM_ETC/controlDict" << endl; } + if (ioRanks_.size()) + { + // Print a bit of information + stringList ioRanks(Pstream::nProcs()); + if (Pstream::master(comm_)) + { + ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid()); + } + Pstream::gatherList(ioRanks); + + Info<< " IO nodes:" << endl; + forAll(ioRanks, proci) + { + if (!ioRanks[proci].empty()) + { + Info<< " " << ioRanks[proci] << endl; + } + } + } + + if ( regIOobject::fileModificationChecking @@ -227,6 +334,80 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation } +Foam::fileOperations::collatedFileOperation::collatedFileOperation +( + const label comm, + const labelList& ioRanks, + const word& typeName, + const bool verbose +) +: + masterUncollatedFileOperation(comm, false), + myComm_(-1), + writer_(maxThreadFileBufferSize, comm), + nProcs_(Pstream::nProcs()), + ioRanks_(ioRanks) +{ + if (verbose) + { + Info<< "I/O : " << typeName + << " (maxThreadFileBufferSize " << maxThreadFileBufferSize + << ')' << endl; + + if (maxThreadFileBufferSize == 0) + { + Info<< " Threading not activated " + "since maxThreadFileBufferSize = 0." << nl + << " Writing may run slowly for large file sizes." + << endl; + } + else + { + Info<< " Threading activated " + "since maxThreadFileBufferSize > 0." << nl + << " Requires large enough buffer to collect all data" + " or thread support " << nl + << " enabled in MPI. If thread support cannot be " + "enabled, deactivate" << nl + << " threading by setting maxThreadFileBufferSize " + "to 0 in the OpenFOAM etc/controlDict" << nl + << endl; + } + + if + ( + regIOobject::fileModificationChecking + == regIOobject::inotifyMaster + ) + { + WarningInFunction + << "Resetting fileModificationChecking to inotify" << endl; + } + + if + ( + regIOobject::fileModificationChecking + == regIOobject::timeStampMaster + ) + { + WarningInFunction + << "Resetting fileModificationChecking to timeStamp" << endl; + } + } +} + + +// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // + +Foam::fileOperations::collatedFileOperation::~collatedFileOperation() +{ + if (myComm_ != -1 && myComm_ != UPstream::worldComm) + { + UPstream::freeCommunicator(myComm_); + } +} + + // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // Foam::fileName Foam::fileOperations::collatedFileOperation::objectPath @@ -238,19 +419,21 @@ Foam::fileName Foam::fileOperations::collatedFileOperation::objectPath // Replacement for objectPath if (io.time().processorCase()) { - return masterUncollatedFileOperation::objectPath + return masterUncollatedFileOperation::localObjectPath ( io, - fileOperation::PROCESSORSOBJECT, + fileOperation::PROCOBJECT, + "dummy", // not used for processorsobject io.instance() ); } else { - return masterUncollatedFileOperation::objectPath + return masterUncollatedFileOperation::localObjectPath ( io, fileOperation::OBJECT, + word::null, io.instance() ); } @@ -276,8 +459,8 @@ bool Foam::fileOperations::collatedFileOperation::writeObject if (debug) { - Pout<< "writeObject:" - << " : For object : " << io.name() + Pout<< "collatedFileOperation::writeObject :" + << " For object : " << io.name() << " falling back to master-only output to " << io.path() << endl; } @@ -313,7 +496,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject else { // Construct the equivalent processors/ directory - fileName path(processorsPath(io, inst)); + fileName path(processorsPath(io, inst, processorsDir(io))); mkDir(path); fileName pathName(path/io.name()); @@ -322,7 +505,8 @@ bool Foam::fileOperations::collatedFileOperation::writeObject { if (debug) { - Pout<< "writeObject:" << " : For global object : " << io.name() + Pout<< "collatedFileOperation::writeObject :" + << " For global object : " << io.name() << " falling back to master-only output to " << pathName << endl; } @@ -359,11 +543,11 @@ bool Foam::fileOperations::collatedFileOperation::writeObject else if (!Pstream::parRun()) { // Special path for e.g. decomposePar. Append to - // processors/ file + // processorsDDD/ file if (debug) { - Pout<< "writeObject:" - << " : For object : " << io.name() + Pout<< "collatedFileOperation::writeObject :" + << " For object : " << io.name() << " appending to " << pathName << endl; } @@ -373,8 +557,8 @@ bool Foam::fileOperations::collatedFileOperation::writeObject { if (debug) { - Pout<< "writeObject:" - << " : For object : " << io.name() + Pout<< "collatedFileOperation::writeObject :" + << " For object : " << io.name() << " starting collating output to " << pathName << endl; } @@ -386,7 +570,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject { return false; } - if (Pstream::master() && !io.writeHeader(os)) + if (Pstream::master(comm_) && !io.writeHeader(os)) { return false; } @@ -395,7 +579,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject { return false; } - if (Pstream::master()) + if (Pstream::master(comm_)) { IOobject::writeEndDivider(os); } @@ -406,4 +590,89 @@ bool Foam::fileOperations::collatedFileOperation::writeObject } +Foam::word Foam::fileOperations::collatedFileOperation::processorsDir +( + const fileName& fName +) const +{ + if (Pstream::parRun()) + { + const List<int>& procs(UPstream::procID(comm_)); + + word procDir(processorsBaseDir+Foam::name(Pstream::nProcs())); + + if (procs.size() != Pstream::nProcs()) + { + procDir += + + "_" + + Foam::name(procs[0]) + + "-" + + Foam::name(procs.last()); + } + return procDir; + } + else + { + word procDir(processorsBaseDir+Foam::name(nProcs_)); + + if (ioRanks_.size()) + { + // Detect current processor number + label proci = detectProcessorPath(fName); + + if (proci != -1) + { + // Find lowest io rank + label minProc = 0; + label maxProc = nProcs_-1; + forAll(ioRanks_, i) + { + if (ioRanks_[i] >= nProcs_) + { + break; + } + else if (ioRanks_[i] <= proci) + { + minProc = ioRanks_[i]; + } + else + { + maxProc = ioRanks_[i]-1; + break; + } + } + procDir += + + "_" + + Foam::name(minProc) + + "-" + + Foam::name(maxProc); + } + } + + return procDir; + } +} + + +Foam::word Foam::fileOperations::collatedFileOperation::processorsDir +( + const IOobject& io +) const +{ + return processorsDir(io.objectPath()); +} + + +void Foam::fileOperations::collatedFileOperation::setNProcs(const label nProcs) +{ + nProcs_ = nProcs; + + if (debug) + { + Pout<< "collatedFileOperation::setNProcs :" + << " Setting number of processors to " << nProcs_ << endl; + } +} + + // ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H index 97eca7d935c5a13816d7f26ff511b1a931f8f3d8..2625deb17568dcd3364a251901bdf121fa3b1e15 100644 --- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H @@ -43,6 +43,7 @@ SourceFiles #include "masterUncollatedFileOperation.H" #include "OFstreamCollator.H" +#include "fileOperationInitialise.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -59,14 +60,33 @@ class collatedFileOperation : public masterUncollatedFileOperation { - // Private data +protected: + + // Protected data + + //- Any communicator allocated by me + const label myComm_; //- Threaded writer mutable OFstreamCollator writer_; + // For non-parallel operation + + //- Number of processors (overall) + label nProcs_; + + //- Ranks of IO handlers + const labelList ioRanks_; + // Private Member Functions + static labelList ioRanks(); + + //- Is proci master of communicator (in parallel) or master of + // the io ranks (non-parallel) + bool isMasterRank(const label proci) const; + //- Append to processors/ file bool appendObject ( @@ -97,9 +117,18 @@ public: //- Construct null collatedFileOperation(const bool verbose); + //- Construct from user communicator + collatedFileOperation + ( + const label comm, + const labelList& ioRanks, + const word& typeName, + const bool verbose + ); + //- Destructor - virtual ~collatedFileOperation() = default; + virtual ~collatedFileOperation(); // Member Functions @@ -123,6 +152,54 @@ public: IOstream::compressionType compression=IOstream::UNCOMPRESSED, const bool valid = true ) const; + + // Other + + //- Actual name of processors dir + virtual word processorsDir(const IOobject&) const; + + //- Actual name of processors dir + virtual word processorsDir(const fileName&) const; + + //- Set number of processor directories/results. Only used in + // decomposePar + virtual void setNProcs(const label nProcs); +}; + + +/*---------------------------------------------------------------------------*\ + Class collatedFileOperationInitialise Declaration +\*---------------------------------------------------------------------------*/ + +class collatedFileOperationInitialise +: + public masterUncollatedFileOperationInitialise +{ +public: + + // Constructors + + //- Construct from components + collatedFileOperationInitialise(int& argc, char**& argv) + : + masterUncollatedFileOperationInitialise(argc, argv) + {} + + + //- Destructor + virtual ~collatedFileOperationInitialise() + {} + + + // Member Functions + + //- Needs threading + virtual bool needsThreading() const + { + return + collatedFileOperation::maxThreadFileBufferSize + > 0; + } }; diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C new file mode 100644 index 0000000000000000000000000000000000000000..e652f21a6718a15c03eaaad0f2cdc360650bb5f5 --- /dev/null +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C @@ -0,0 +1,176 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +\*---------------------------------------------------------------------------*/ + +#include "hostCollatedFileOperation.H" +#include "addToRunTimeSelectionTable.H" +#include "bitSet.H" + +/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */ + +namespace Foam +{ +namespace fileOperations +{ + defineTypeNameAndDebug(hostCollatedFileOperation, 0); + addToRunTimeSelectionTable + ( + fileOperation, + hostCollatedFileOperation, + word + ); + + // Register initialisation routine. Signals need for threaded mpi and + // handles command line arguments + addNamedToRunTimeSelectionTable + ( + fileOperationInitialise, + hostCollatedFileOperationInitialise, + word, + hostCollated + ); +} +} + + +// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // + +Foam::labelList Foam::fileOperations::hostCollatedFileOperation::subRanks +( + const label n +) +{ + DynamicList<label> subRanks(64); + + string ioRanksString(getEnv("FOAM_IORANKS")); + if (!ioRanksString.empty()) + { + IStringStream is(ioRanksString); + labelList ioRanks(is); + + if (findIndex(ioRanks, 0) == -1) + { + FatalErrorInFunction + << "Rank 0 (master) should be in the IO ranks. Currently " + << ioRanks << exit(FatalError); + } + + // The lowest numbered rank is the IO rank + const bitSet isIOrank(n, ioRanks); + + for (label proci = Pstream::myProcNo(); proci >= 0; --proci) + { + if (isIOrank[proci]) + { + // Found my master. Collect all processors with same master + subRanks.append(proci); + for + ( + label rank = proci+1; + rank < n && !isIOrank[rank]; + ++rank + ) + { + subRanks.append(rank); + } + break; + } + } + } + else + { + // Normal operation: one lowest rank per hostname is the writer + const string myHostName(hostName()); + + stringList hosts(Pstream::nProcs()); + hosts[Pstream::myProcNo()] = myHostName; + Pstream::gatherList(hosts); + Pstream::scatterList(hosts); + + // Collect procs with same hostname + forAll(hosts, proci) + { + if (hosts[proci] == myHostName) + { + subRanks.append(proci); + } + } + } + return subRanks; +} + + +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation +( + const bool verbose +) +: + collatedFileOperation + ( + UPstream::allocateCommunicator + ( + UPstream::worldComm, + subRanks(Pstream::nProcs()) + ), + (Pstream::parRun() ? labelList(0) : ioRanks()), // processor dirs + typeName, + verbose + ) +{ + if (verbose) + { + // Print a bit of information + stringList ioRanks(Pstream::nProcs()); + if (Pstream::master(comm_)) + { + ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid()); + } + Pstream::gatherList(ioRanks); + + Info<< " IO nodes:" << endl; + forAll(ioRanks, proci) + { + if (!ioRanks[proci].empty()) + { + Info<< " " << ioRanks[proci] << endl; + } + } + } +} + + +// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // + +Foam::fileOperations::hostCollatedFileOperation::~hostCollatedFileOperation() +{ + if (comm_ != -1 && comm_ != UPstream::worldComm) + { + UPstream::freeCommunicator(comm_); + } +} + + +// ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H new file mode 100644 index 0000000000000000000000000000000000000000..4d61b622b241602208d1391d8bcf7c2e6d3dcf91 --- /dev/null +++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H @@ -0,0 +1,134 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +Class + Foam::fileOperations::hostCollatedFileOperation + +Description + Version of collatedFileOperation with multiple read/write ranks. + + In parallel it will assume ranks are sorted according to hostname + and the lowest rank per hostname will be the IO rank. The output directories + will get a unique name processors<N>_<low>-<high> where N is the overall + number of processors and low and high is the range of ranks contained + in the files. Each of these subsets uses its own communicator. + + Instead of using the hostnames the IO ranks can be assigned using the + FOAM_IORANKS environment variable (also when running non-parallel), e.g. + when decomposing into 4: + + FOAM_IORANKS='(0 2)' decomposePar -fileHandler hostCollated + + will generate + + processors4_0-1/ + containing data for processors 0 to 1 + processors4_2-3/ + containing data for processors 2 to 3 + +See also + collatedFileOperation + +SourceFiles + hostCollatedFileOperation.C + +\*---------------------------------------------------------------------------*/ + +#ifndef fileOperations_hostCollatedFileOperation_H +#define fileOperations_hostCollatedFileOperation_H + +#include "collatedFileOperation.H" + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ +namespace fileOperations +{ + +/*---------------------------------------------------------------------------*\ + Class hostCollatedFileOperation Declaration +\*---------------------------------------------------------------------------*/ + +class hostCollatedFileOperation +: + public collatedFileOperation +{ + // Private Member Functions + + //- Get the list of processors part of this set + static labelList subRanks(const label n); + + +public: + + //- Runtime type information + TypeName("hostCollated"); + + + // Constructors + + //- Construct null + hostCollatedFileOperation(const bool verbose); + + + //- Destructor + virtual ~hostCollatedFileOperation(); +}; + + +/*---------------------------------------------------------------------------*\ + Class hostCollatedFileOperationInitialise Declaration +\*---------------------------------------------------------------------------*/ + +class hostCollatedFileOperationInitialise +: + public collatedFileOperationInitialise +{ +public: + + // Constructors + + //- Construct from components + hostCollatedFileOperationInitialise(int& argc, char**& argv) + : + collatedFileOperationInitialise(argc, argv) + {} + + + //- Destructor + virtual ~hostCollatedFileOperationInitialise() + {} +}; + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace fileOperations +} // End namespace Foam + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#endif + +// ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C index 93e007ad4a4ca56b07dd410a4e56872585c1577b..999562fa03e584676a68f332ca303eabaa6abd13 100644 --- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C +++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -28,11 +28,11 @@ License #include "regIOobject.H" #include "argList.H" #include "HashSet.H" -#include "masterUncollatedFileOperation.H" #include "objectRegistry.H" #include "decomposedBlockData.H" #include "polyMesh.H" #include "registerSwitch.H" +#include "Time.H" /* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */ @@ -43,6 +43,28 @@ namespace Foam defineTypeNameAndDebug(fileOperation, 0); defineRunTimeSelectionTable(fileOperation, word); + template<> + const char* Foam::NamedEnum + < + fileOperation::pathType, + 12 + >::names[] = + { + "notFound", + "absolute", + "objectPath", + "writeObject", + "uncollatedProc", + "globalProc", + "localProc", + "parentObjectPath", + "findInstance", + "uncollatedProcInstance", + "globalProcInstance", + "localProcInstance" + }; + const NamedEnum<fileOperation::pathType, 12> fileOperation::pathTypeNames_; + word fileOperation::defaultFileHandler ( debug::optimisationSwitches().lookupOrAddDefault @@ -56,7 +78,7 @@ namespace Foam ); } -Foam::word Foam::fileOperation::processorsDir = "processors"; +Foam::word Foam::fileOperation::processorsBaseDir = "processors"; // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // @@ -133,6 +155,80 @@ Foam::instantList Foam::fileOperation::sortTimes } +void Foam::fileOperation::mergeTimes +( + const instantList& extraTimes, + const word& constantName, + instantList& times +) +{ + if (extraTimes.size()) + { + bool haveConstant = + ( + times.size() > 0 + && times[0].name() == constantName + ); + + bool haveExtraConstant = + ( + extraTimes.size() > 0 + && extraTimes[0].name() == constantName + ); + + // Combine times + instantList combinedTimes(times.size()+extraTimes.size()); + label sz = 0; + label extrai = 0; + if (haveExtraConstant) + { + extrai = 1; + if (!haveConstant) + { + combinedTimes[sz++] = extraTimes[0]; // constant + } + } + forAll(times, i) + { + combinedTimes[sz++] = times[i]; + } + for (; extrai < extraTimes.size(); extrai++) + { + combinedTimes[sz++] = extraTimes[extrai]; + } + combinedTimes.setSize(sz); + times.transfer(combinedTimes); + + // Sort + if (times.size() > 1) + { + label starti = 0; + if (times[0].name() == constantName) + { + starti = 1; + } + std::sort(×[starti], times.end(), instant::less()); + + // Filter out duplicates + label newi = starti+1; + for (label i = newi; i < times.size(); i++) + { + if (times[i].value() != times[i-1].value()) + { + if (newi != i) + { + times[newi] = times[i]; + } + newi++; + } + } + + times.setSize(newi); + } + } +} + + bool Foam::fileOperation::isFileOrDir(const bool isFile, const fileName& f) { return @@ -141,8 +237,182 @@ bool Foam::fileOperation::isFileOrDir(const bool isFile, const fileName& f) } +Foam::tmpNrc<Foam::fileOperation::dirIndexList> +Foam::fileOperation::lookupProcessorsPath(const fileName& fName) const +{ + // If path is local to a processor (e.g. contains 'processor2') + // find the corresponding actual processor directory (e.g. 'processors4') + // and index (2) + + fileName path; + fileName pDir; + fileName local; + label gStart; + label gSz; + label numProcs; + label proci = + splitProcessorPath(fName, path, pDir, local, gStart, gSz, numProcs); + + if (proci != -1) + { + const fileName procPath(path/pDir); + + HashTable<dirIndexList>::const_iterator iter = + procsDirs_.find(procPath); + + if (iter != procsDirs_.end()) + { + return iter(); + } + + // Read all directories to see any beginning with processor + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + DynamicList<dirIndex> procDirs; + + // Note: use parallel synchronised reading so cache will be same + // order on all processors + fileNameList dirNames(readDir(path, fileName::Type::DIRECTORY)); + + // Extract info from processorsDDD or processorDDD: + // - highest processor number + // - directory+offset containing data for proci + label maxProc = -1; + + forAll(dirNames, i) + { + const fileName& dirN = dirNames[i]; + + // Analyse directory name + fileName rp, rd, rl; + label rStart, rSize, rNum; + label readProci = + splitProcessorPath(dirN, rp, rd, rl, rStart, rSize, rNum); + maxProc = max(maxProc, readProci); + + if (proci == readProci) + { + // Found "processorDDD". No need for index. + procDirs.append + ( + dirIndex + ( + dirN, + Tuple2<pathType, label>(PROCUNCOLLATED, -1) + ) + ); + } + else if (proci >= rStart && proci < rStart+rSize) + { + // "processorsDDD_start-end" + // Found the file that contains the data for proci + procDirs.append + ( + dirIndex + ( + dirN, + Tuple2<pathType, label>(PROCOBJECT, proci-rStart) + ) + ); + } + if (rNum != -1) + { + // Direct detection of processorsDDD + maxProc = rNum-1; + + if (rStart == -1) + { + // "processorsDDD" + procDirs.append + ( + dirIndex + ( + dirN, + Tuple2<pathType, label>(PROCBASEOBJECT, proci) + ) + ); + } + } + } + if (!Pstream::parRun()) + { + // If (as a side effect) we found the number of decompositions + // use it + if (maxProc != -1) + { + const_cast<fileOperation&>(*this).setNProcs(maxProc+1); + } + } + + if (returnReduce(procDirs.size(), sumOp<label>())) + { + procsDirs_.insert(procPath, procDirs); + + if (debug) + { + Pout<< "fileOperation::lookupProcessorsPath : For:" << procPath + << " detected:" << procDirs << endl; + } + + // Make sure to return a reference + return procsDirs_[procPath]; + } + } + return tmpNrc<dirIndexList>(new dirIndexList(0, dirIndex())); +} + + +bool Foam::fileOperation::exists(IOobject& io) const +{ + // Generate output filename for object + fileName objPath(objectPath(io, word::null)); + + // Test for either directory or a (valid) file & IOobject + bool ok; + if (io.name().empty()) + { + ok = isDir(objPath); + } + else + { + ok = + isFile(objPath) + && io.typeHeaderOk<IOList<label>>(false);// object with local scope + } + + if (!ok) + { + // Re-test with searched for objectPath. This is for backwards + // compatibility + fileName originalPath(filePath(io.objectPath())); + if (originalPath != objPath) + { + // Test for either directory or a (valid) file & IOobject + if (io.name().empty()) + { + ok = isDir(originalPath); + } + else + { + ok = + isFile(originalPath) + && io.typeHeaderOk<IOList<label>>(false); + } + } + } + + return ok; +} + + // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // +Foam::fileOperation::fileOperation(label comm) +: + comm_(comm) +{} + + Foam::autoPtr<Foam::fileOperation> Foam::fileOperation::New ( const word& handlerType, @@ -238,37 +508,64 @@ bool Foam::fileOperation::writeObject } -//Foam::fileName Foam::fileOperation::objectPath(const fileName& fName) const -//{ -// return fName; -//} - - Foam::fileName Foam::fileOperation::filePath(const fileName& fName) const { + if (debug) + { + Pout<< "fileOperation::filePath :" << " fName:" << fName << endl; + } + fileName path; + fileName pDir; fileName local; - label proci = fileOperations::masterUncollatedFileOperation:: - splitProcessorPath - ( - fName, - path, - local - ); + label gStart; + label gSz; + label numProcs; + label proci = + splitProcessorPath(fName, path, pDir, local, gStart, gSz, numProcs); - fileName procsName(path/processorsDir/local); + if (numProcs != -1) + { + WarningInFunction << "Filename is already adapted:" << fName << endl; + } // Give preference to processors variant - if (proci != -1 && exists(procsName)) + if (proci != -1) { - return procsName; + // Get all processor directories + tmpNrc<dirIndexList> procDirs(lookupProcessorsPath(fName)); + forAll(procDirs(), i) + { + const fileName& procDir = procDirs()[i].first(); + + fileName collatedName(path/procDir/local); + if (exists(collatedName)) + { + if (debug) + { + Pout<< "fileOperation::filePath : " << collatedName << endl; + } + return collatedName; + } + } } - else if (exists(fName)) + + if (exists(fName)) { + if (debug) + { + Pout<< "fileOperation::filePath : " << fName << endl; + } return fName; } - - return fileName::null; + else + { + if (debug) + { + Pout<< "fileOperation::filePath : Not found" << endl; + } + return fileName::null; + } } @@ -377,8 +674,8 @@ Foam::instantList Foam::fileOperation::findTimes { if (debug) { - Pout<< FUNCTION_NAME - << " : Finding times in directory " << directory << endl; + Pout<< "fileOperation::findTimes : Finding times in directory " + << directory << endl; } // Read directory entries into a list @@ -393,100 +690,185 @@ Foam::instantList Foam::fileOperation::findTimes instantList times = sortTimes(dirEntries, constantName); - // Check if directory is processorXXX - fileName procsDir - ( - fileOperations::masterUncollatedFileOperation::processorsPath - ( - directory - ) - ); - if (!procsDir.empty() && procsDir != directory) + // Get all processor directories + tmpNrc<dirIndexList> procDirs(lookupProcessorsPath(directory)); + forAll(procDirs(), i) { - fileNameList extraEntries - ( - Foam::readDir - ( - procsDir, - fileName::DIRECTORY - ) - ); - - instantList extraTimes = sortTimes(extraEntries, constantName); - - if (extraTimes.size()) + const fileName& procDir = procDirs()[i].first(); + fileName collDir(processorsPath(directory, procDir)); + if (!collDir.empty() && collDir != directory) { - bool haveConstant = + fileNameList extraEntries ( - times.size() > 0 - && times[0].name() == constantName + Foam::readDir + ( + collDir, + fileName::DIRECTORY + ) ); - - bool haveExtraConstant = + mergeTimes ( - extraTimes.size() > 0 - && extraTimes[0].name() == constantName + sortTimes(extraEntries, constantName), + constantName, + times ); + } + } - // Combine times - instantList combinedTimes(times.size()+extraTimes.size()); - label sz = 0; - label extrai = 0; - if (haveExtraConstant) - { - extrai = 1; - if (!haveConstant) - { - combinedTimes[sz++] = extraTimes[0]; // constant - } - } - forAll(times, i) + if (debug) + { + Pout<< "fileOperation::findTimes : Found times:" << times << endl; + } + return times; +} + + +Foam::IOobject Foam::fileOperation::findInstance +( + const IOobject& startIO, + const scalar startValue, + const word& stopInstance +) const +{ + const Time& time = startIO.time(); + + IOobject io(startIO); + + // Note: - if name is empty, just check the directory itself + // - check both for isFile and headerOk since the latter does a + // filePath so searches for the file. + // - check for an object with local file scope (so no looking up in + // parent directory in case of parallel) + + if (exists(io)) + { + if (debug) + { + InfoInFunction + << "Found exact match for \"" << io.name() + << "\" in " << io.instance()/io.local() + << endl; + } + + return io; + } + + // Search back through the time directories to find the time + // closest to and lower than current time + + instantList ts = time.times(); + label instanceI; + + for (instanceI = ts.size()-1; instanceI >= 0; --instanceI) + { + if (ts[instanceI].value() <= startValue) + { + break; + } + } + + // continue searching from here + for (; instanceI >= 0; --instanceI) + { + // Shortcut: if actual directory is the timeName we've already tested it + if (ts[instanceI].name() == startIO.instance()) + { + continue; + } + + io.instance() = ts[instanceI].name(); + if (exists(io)) + { + if (debug) { - combinedTimes[sz++] = times[i]; + InfoInFunction + << "Found exact match for \"" << io.name() + << "\" in " << io.instance()/io.local() + << endl; } - for (; extrai < extraTimes.size(); extrai++) + + return io; + } + + // Check if hit minimum instance + if (ts[instanceI].name() == stopInstance) + { + if (debug) { - combinedTimes[sz++] = extraTimes[extrai]; + InfoInFunction + << "Hit stopInstance " << stopInstance << endl; } - combinedTimes.setSize(sz); - times.transfer(combinedTimes); - // Sort - if (times.size() > 1) + if + ( + startIO.readOpt() == IOobject::MUST_READ + || startIO.readOpt() == IOobject::MUST_READ_IF_MODIFIED + ) { - label starti = 0; - if (times[0].name() == constantName) + if (io.name().empty()) { - starti = 1; + FatalErrorInFunction + << "Cannot find directory " + << io.local() << " in times " << startIO.instance() + << " down to " << stopInstance + << exit(FatalError); } - std::sort(×[starti], times.end(), instant::less()); - - // Filter out duplicates - label newi = starti+1; - for (label i = newi; i < times.size(); i++) + else { - if (times[i].value() != times[i-1].value()) - { - if (newi != i) - { - times[newi] = times[i]; - } - newi++; - } + FatalErrorInFunction + << "Cannot find file \"" << io.name() + << "\" in directory " << io.local() + << " in times " << startIO.instance() + << " down to " << stopInstance + << exit(FatalError); } + } + + return io; + } + } - times.setSize(newi); + // times() usually already includes the constant() so would have been + // checked above. Re-test if + // - times() is empty. Sometimes this can happen (e.g. decomposePar with + // collated) + // - times()[0] is not constant + if (!ts.size() || ts[0].name() != time.constant()) + { + // Note. This needs to be a hard-coded constant, rather than the + // constant function of the time, because the latter points to + // the case constant directory in parallel cases + + io.instance() = time.constant(); + if (exists(io)) + { + if (debug) + { + InfoInFunction + << "Found constant match for \"" << io.name() + << "\" in " << io.instance()/io.local() + << endl; } + return io; } } - if (debug) + + if + ( + startIO.readOpt() == IOobject::MUST_READ + || startIO.readOpt() == IOobject::MUST_READ_IF_MODIFIED + ) { - Pout<< FUNCTION_NAME - << " : Found times:" << times << endl; + FatalErrorInFunction + << "Cannot find file \"" << io.name() << "\" in directory " + << io.local() << " in times " << startIO.instance() + << " down to " << time.constant() + << exit(FatalError); } - return times; + + return io; } @@ -518,19 +900,9 @@ Foam::fileNameList Foam::fileOperation::readObjects else { // Get processors equivalent of path + fileName procsPath(filePath(path)); - fileName prefix; - fileName postfix; - label proci = fileOperations::masterUncollatedFileOperation:: - splitProcessorPath - ( - path, - prefix, - postfix - ); - fileName procsPath(prefix/processorsDir/postfix); - - if (proci != -1 && Foam::isDir(procsPath)) + if (!procsPath.empty()) { newInstance = instance; objectNames = Foam::readDir(procsPath, fileName::FILE); @@ -540,46 +912,252 @@ Foam::fileNameList Foam::fileOperation::readObjects } +void Foam::fileOperation::setNProcs(const label nProcs) +{} + + Foam::label Foam::fileOperation::nProcs ( const fileName& dir, const fileName& local ) const { - if (Foam::isDir(dir/processorsDir)) + label nProcs = 0; + if (Pstream::master(comm_)) { - fileName pointsFile - ( - dir - /processorsDir - /"constant" - /local - /polyMesh::meshSubDir - /"points" - ); + fileNameList dirNames(Foam::readDir(dir, fileName::Type::DIRECTORY)); - if (Foam::isFile(pointsFile)) + // Detect any processorsDDD or processorDDD + label maxProc = -1; + forAll(dirNames, i) { - return decomposedBlockData::numBlocks(pointsFile); + const fileName& dirN = dirNames[i]; + + fileName path, pDir, local; + label start, size, n; + maxProc = max + ( + maxProc, + splitProcessorPath(dirN, path, pDir, local, start, size, n) + ); + if (n != -1) + { + // Direct detection of processorsDDD + maxProc = n-1; + break; + } } - else + nProcs = maxProc+1; + + + if (nProcs == 0 && Foam::isDir(dir/processorsBaseDir)) { - WarningInFunction << "Cannot read file " << pointsFile - << " to determine the number of decompositions." - << " Falling back to looking for processor.*" << endl; + fileName pointsFile + ( + dir + /processorsBaseDir + /"constant" + /local + /polyMesh::meshSubDir + /"points" + ); + + if (Foam::isFile(pointsFile)) + { + nProcs = decomposedBlockData::numBlocks(pointsFile); + } + else + { + WarningInFunction << "Cannot read file " << pointsFile + << " to determine the number of decompositions." + << " Returning 1" << endl; + } } } + Pstream::scatter(nProcs, Pstream::msgType(), comm_); + return nProcs; +} - label nProcs = 0; - while - ( - isDir(dir/(word("processor") + name(nProcs))) - ) + +Foam::fileName Foam::fileOperation::processorsCasePath +( + const IOobject& io, + const word& procsDir +) const +{ + return io.rootPath()/io.time().globalCaseName()/procsDir; +} + + +Foam::fileName Foam::fileOperation::processorsPath +( + const IOobject& io, + const word& instance, + const word& procsDir +) const +{ + return + processorsCasePath(io, procsDir) + /instance + /io.db().dbDir() + /io.local(); +} + + +Foam::fileName Foam::fileOperation::processorsPath +( + const fileName& dir, + const word& procsDir +) const +{ + // Check if directory is processorDDD + word caseName(dir.name()); + + std::string::size_type pos = caseName.find("processor"); + if (pos == 0) { - ++nProcs; + if (caseName.size() <= 9 || caseName[9] == 's') + { + WarningInFunction << "Directory " << dir + << " does not end in old-style processorDDD" << endl; + } + + return dir.path()/procsDir; } + else + { + return fileName::null; + } +} - return nProcs; + +Foam::label Foam::fileOperation::splitProcessorPath +( + const fileName& objectPath, + fileName& path, + fileName& procDir, + fileName& local, + + label& groupStart, + label& groupSize, + + label& nProcs +) +{ + path.clear(); + procDir.clear(); + local.clear(); + + // Potentially detected start of number of processors in local group + groupStart = -1; + groupSize = 0; + + // Potentially detected number of processors + nProcs = -1; + + // Search for processor at start of line or /processor + std::string::size_type pos = objectPath.find("processor"); + if (pos == string::npos) + { + return -1; + } + + // "processorDDD" + // "processorsNNN" + // "processorsNNN_AA-BB" + + + if (pos > 0 && objectPath[pos-1] != '/') + { + // Directory not starting with "processor" e.g. "somenamewithprocessor" + return -1; + } + + procDir = objectPath; + + // Strip leading directory + if (pos > 0) + { + path = objectPath.substr(0, pos-1); + procDir = objectPath.substr(pos); + } + + // Strip trailing local directory + pos = procDir.find('/'); + if (pos != string::npos) + { + local = procDir.substr(pos+1); + procDir = procDir.substr(0, pos); + } + + // Now procDir is e.g. + // - processor0 + // - processors0 + // - processorBananas + + // Look for number after "processor" + + fileName f(procDir.substr(9)); + + if (f.size() && f[0] == 's') + { + // "processsorsNNN" + + f = f.substr(1); + + // Detect "processorsNNN_AA-BB" + { + std::string::size_type fromStart = f.find("_"); + std::string::size_type toStart = f.find("-"); + if (fromStart != string::npos && toStart != string::npos) + { + string nProcsName(f.substr(0, fromStart)); + string fromName(f.substr(fromStart+1, toStart-(fromStart+1))); + string toName(f.substr(toStart+1)); + + label groupEnd = -1; + if + ( + Foam::read(fromName.c_str(), groupStart) + && Foam::read(toName.c_str(), groupEnd) + && Foam::read(nProcsName.c_str(), nProcs) + ) + { + groupSize = groupEnd-groupStart+1; + return -1; + } + } + } + + // Detect "processorsN" + label n; + if (Foam::read(f.c_str(), n)) + { + nProcs = n; + } + return -1; + } + else + { + // Detect "processorN" + label proci; + if (Foam::read(f.c_str(), proci)) + { + return proci; + } + else + { + return -1; + } + } +} + + +Foam::label Foam::fileOperation::detectProcessorPath(const fileName& fName) +{ + fileName path, pDir, local; + label start, size, nProcs; + return splitProcessorPath(fName, path, pDir, local, start, size, nProcs); } diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H index b075e6cec5982dbf23d39993930ad4364d1cf805..f3887fc056dc8dbd94514e51d6c5bd5f7ae3ac54 100644 --- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H +++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H @@ -45,6 +45,10 @@ Description #include "instantList.H" #include "fileMonitor.H" #include "labelList.H" +#include "Switch.H" +#include "tmpNrc.H" +#include "NamedEnum.H" +#include "Tuple2.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -56,16 +60,55 @@ class regIOobject; class objectRegistry; class Time; +// Description of processor directory naming: +// - processor directory naming +// - whether directory contains a range (so differs on different processors) +// - index in range +//typedef Tuple2<fileName, Tuple2<bool, label>> dirIndex; +//typedef List<dirIndex> dirIndexList; + /*---------------------------------------------------------------------------*\ Class fileOperation Declaration \*---------------------------------------------------------------------------*/ class fileOperation { +public: + + //- Enumeration for the location of an IOobject + enum pathType + { + NOTFOUND, // not found + ABSOLUTE, // instance is absolute directory + OBJECT, // io.objectPath() exists + WRITEOBJECT, // write path exists + PROCUNCOLLATED, // objectPath exists in processor0 + PROCBASEOBJECT, // objectPath exists in specified, constant + // processorsDir (usually 'processorsDDD') + PROCOBJECT, // objectPath exists in locally differing + // processorsDir (e.g. 'processorsDDD_0-1') + PARENTOBJECT, // parent of object path + FINDINSTANCE, // file found in time directory + PROCUNCOLLATEDINSTANCE, // as PROCUNCOLLATED but with instance + PROCBASEINSTANCE, // as PROCBASEOBJECT but with instance + PROCINSTANCE // as PROCOBJECT but with instance + }; + static const NamedEnum<pathType, 12> pathTypeNames_; + + typedef Tuple2<fileName, Tuple2<pathType, label>> dirIndex; + typedef List<dirIndex> dirIndexList; + + protected: // Protected data + //- Communicator to use + const label comm_; + + //- Detected processors directories + mutable HashTable<dirIndexList> procsDirs_; + //- file-change monitor for all registered files mutable autoPtr<fileMonitor> monitorPtr_; @@ -77,15 +120,35 @@ protected: //- Sort directory entries according to time value static instantList sortTimes(const fileNameList&, const word&); + //- Merge two times + static void mergeTimes + ( + const instantList& extraTimes, + const word& constantName, + instantList& times + ); + //- Helper: check for file (isFile) or directory (!isFile) static bool isFileOrDir(const bool isFile, const fileName&); + //- Detect presence of processorsDDD + void cacheProcessorsPath(const fileName& fName) const; + + //- Lookup name of processorsDDD using cache. Return empty fileName + // if not found + tmpNrc<dirIndexList> lookupProcessorsPath(const fileName&) const; + + //- Does ioobject exist. Is either a directory (empty name()) or + // a file + bool exists(IOobject& io) const; + + public: // Static data - //- The processors directory name (usually "processors") - static word processorsDir; + //- Return the processors directory name (usually "processors") + static word processorsBaseDir; //- Default fileHandler static word defaultFileHandler; @@ -93,19 +156,6 @@ public: // Public data types - //- Enumeration for the location of an IOobject - enum pathType - { - NOTFOUND, // not found - ABSOLUTE, // instance is absolute directory - OBJECT, // objectPath exists - PROCESSORSOBJECT, // objectPath exists in processors/ - PARENTOBJECT, // parent of object path - FINDINSTANCE, // file found in time directory - PROCESSORSFINDINSTANCE // as above but in processors/ - }; - - //- Runtime type information TypeName("fileOperation"); @@ -116,8 +166,8 @@ public: // Constructors - //- Construct null - fileOperation() = default; + //- Construct from communicator + explicit fileOperation(const label comm); // Declare run-time constructor selection table @@ -413,6 +463,24 @@ public: // Other + //- Actual name of processors dir (for use in mode PROCOBJECT, + // PROCINSTANCE) + virtual word processorsDir(const IOobject& io) const + { + return processorsBaseDir; + } + + //- Actual name of processors dir (for use in mode PROCOBJECT, + // PROCINSTANCE) + virtual word processorsDir(const fileName&) const + { + return processorsBaseDir; + } + + //- Set number of processor directories/results. Only used in + // decomposePar + virtual void setNProcs(const label nProcs); + //- Get number of processor directories/results. Used for e.g. // reconstructPar, argList checking virtual label nProcs @@ -424,9 +492,59 @@ public: //- Get sorted list of times virtual instantList findTimes(const fileName&, const word&) const; + //- Find instance where IOobject is. Fails if cannot be found + // and readOpt() is MUST_READ/MUST_READ_IF_MODIFIED. Otherwise + // returns stopInstance. + virtual IOobject findInstance + ( + const IOobject& io, + const scalar startValue, + const word& stopInstance + ) const; + //- Callback for time change virtual void setTime(const Time&) const {} + + //- Generate path (like io.path) from root+casename with any + // 'processorXXX' replaced by procDir (usually 'processsors') + fileName processorsCasePath + ( + const IOobject&, + const word& procDir + ) const; + + //- Generate path (like io.path) with provided instance and any + // 'processorXXX' replaced by procDir (usually 'processsors') + fileName processorsPath + ( + const IOobject&, + const word& instance, + const word& procDir + ) const; + + //- Operating on fileName: replace processorXXX with procDir + fileName processorsPath(const fileName&, const word& procDir) const; + + //- Split fileName into part before 'processor' and part after. + // Returns -1 or processor number and optionally number + // of processors. Use with care. + // - path/"processor"+Foam::name(proci)/local reconstructs input + // - path/"processors"+Foam::name(nProcs)/local reconstructs + // collated processors equivalence + static label splitProcessorPath + ( + const fileName&, + fileName& path, + fileName& procDir, + fileName& local, + label& groupStart, + label& groupSize, + label& nProcs + ); + + //- Detect processor number from '/aa/bb/processorDDD/cc' + static label detectProcessorPath(const fileName&); }; diff --git a/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.C b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.C new file mode 100644 index 0000000000000000000000000000000000000000..3494fe0ca9167531720c457f1040a8fb670b29d1 --- /dev/null +++ b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.C @@ -0,0 +1,88 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +\*---------------------------------------------------------------------------*/ + +#include "fileOperationInitialise.H" +#include "addToRunTimeSelectionTable.H" +#include "OSspecific.H" + +/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */ + +namespace Foam +{ +namespace fileOperations +{ + defineTypeNameAndDebug(fileOperationInitialise, 0); + defineRunTimeSelectionTable(fileOperationInitialise, word); +} +} + + +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::fileOperations::fileOperationInitialise::fileOperationInitialise +( + int& argc, + char**& argv +) +{} + + +Foam::autoPtr<Foam::fileOperations::fileOperationInitialise> +Foam::fileOperations::fileOperationInitialise::New +( + const word& type, + int& argc, + char**& argv +) +{ + if (debug) + { + InfoInFunction << "Constructing fileOperationInitialise" << endl; + } + + wordConstructorTable::iterator cstrIter = + wordConstructorTablePtr_->find(type); + + if (cstrIter == wordConstructorTablePtr_->end()) + { + FatalErrorInFunction + << "Unknown fileOperationInitialise type " + << type << nl << nl + << "Valid fileOperationInitialise types are" << endl + << wordConstructorTablePtr_->sortedToc() + << abort(FatalError); + } + + return autoPtr<fileOperationInitialise>(cstrIter()(argc, argv)); +} + + +// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // + +Foam::fileOperations::fileOperationInitialise::~fileOperationInitialise() +{} + + +// ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.H b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.H new file mode 100644 index 0000000000000000000000000000000000000000..b7b2c78cb3c70cba78af8f2c438c2b5a9b72d064 --- /dev/null +++ b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.H @@ -0,0 +1,102 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +Class + Foam::fileOperationInitialise + +\*---------------------------------------------------------------------------*/ + +#ifndef fileOperationInitialise_H +#define fileOperationInitialise_H + +#include "runTimeSelectionTables.H" + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ +namespace fileOperations +{ + +/*---------------------------------------------------------------------------*\ + Class fileOperationInitialise Declaration +\*---------------------------------------------------------------------------*/ + +class fileOperationInitialise +{ +public: + + //- Runtime type information + TypeName("fileOperationInitialise"); + + + // Constructors + + //- Construct components + fileOperationInitialise(int& argc, char**& argv); + + + // Declare run-time constructor selection table + + declareRunTimeSelectionTable + ( + autoPtr, + fileOperationInitialise, + word, + ( + int& argc, char**& argv + ), + (argc, argv) + ); + + + // Selectors + + //- Select type + static autoPtr<fileOperationInitialise> New + ( + const word& type, int& argc, char**& argv + ); + + + //- Destructor + virtual ~fileOperationInitialise(); + + + // Member Functions + + //- Needs threading + virtual bool needsThreading() const = 0; +}; + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace fileOperations +} // End namespace Foam + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#endif + +// ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/fileOperationInitialise/unthreadedInitialise.H b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/unthreadedInitialise.H new file mode 100644 index 0000000000000000000000000000000000000000..8c99df8d332ef4fb2e2580e91d3ab325a4f7ca52 --- /dev/null +++ b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/unthreadedInitialise.H @@ -0,0 +1,84 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +Class + Foam::unthreadedInitialise + +\*---------------------------------------------------------------------------*/ + +#ifndef unthreadedInitialise_H +#define unthreadedInitialise_H + +#include "fileOperationInitialise.H" + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ +namespace fileOperations +{ + +/*---------------------------------------------------------------------------*\ + Class unthreadedInitialise Declaration +\*---------------------------------------------------------------------------*/ + +class unthreadedInitialise +: + public fileOperationInitialise +{ +public: + + // Constructors + + //- Construct from components + unthreadedInitialise(int& argc, char**& argv) + : + fileOperationInitialise(argc, argv) + {} + + + //- Destructor + virtual ~unthreadedInitialise() + {} + + + // Member Functions + + //- Needs threading + virtual bool needsThreading() const + { + return false; + } +}; + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace fileOperations +} // End namespace Foam + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#endif + +// ************************************************************************* // diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C index 4d97c0bdad6109b2b339d39250990c1f16499162..b468eccbe285d175b2874bb60d0a51295dac3a6d 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -34,6 +34,8 @@ License #include "registerSwitch.H" #include "dummyISstream.H" #include "SubList.H" +#include "unthreadedInitialise.H" +#include "bitSet.H" /* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */ @@ -59,12 +61,71 @@ namespace fileOperations float, masterUncollatedFileOperation::maxMasterFileBufferSize ); + + // Mark as not needing threaded mpi + addNamedToRunTimeSelectionTable + ( + fileOperationInitialise, + masterUncollatedFileOperationInitialise, + word, + masterUncollated + ); } } // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // +Foam::labelList Foam::fileOperations::masterUncollatedFileOperation::subRanks +( + const label n +) +{ + string ioRanksString(getEnv("FOAM_IORANKS")); + if (ioRanksString.empty()) + { + return identity(n); + } + else + { + DynamicList<label> subRanks(n); + + IStringStream is(ioRanksString); + labelList ioRanks(is); + + if (!ioRanks.found(0)) + { + FatalErrorInFunction + << "Rank 0 (master) should be in the IO ranks. Currently " + << ioRanks << exit(FatalError); + } + + // The lowest numbered rank is the IO rank + const bitSet isIOrank(n, ioRanks); + + for (label proci = Pstream::myProcNo(); proci >= 0; --proci) + { + if (isIOrank[proci]) + { + // Found my master. Collect all processors with same master + subRanks.append(proci); + for + ( + label rank = proci+1; + rank < n && !isIOrank[rank]; + ++rank + ) + { + subRanks.append(rank); + } + break; + } + } + return subRanks; + } +} + + Foam::word Foam::fileOperations::masterUncollatedFileOperation::findInstancePath ( @@ -90,26 +151,29 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstancePath } -Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo +Foam::fileName +Foam::fileOperations::masterUncollatedFileOperation::filePathInfo ( const bool checkGlobal, const bool isFile, const IOobject& io, const bool search, pathType& searchType, + word& procsDir, word& newInstancePath ) const { + procsDir = word::null; newInstancePath = word::null; if (io.instance().isAbsolute()) { - fileName objectPath = io.instance()/io.name(); + fileName objPath = io.instance()/io.name(); - if (isFileOrDir(isFile, objectPath)) + if (isFileOrDir(isFile, objPath)) { searchType = fileOperation::ABSOLUTE; - return objectPath; + return objPath; } else { @@ -119,24 +183,45 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo } else { - // 1. Check processors/ + // 1. Check the writing fileName + fileName writePath(objectPath(io, io.headerClassName())); + + if (isFileOrDir(isFile, writePath)) + { + searchType = fileOperation::WRITEOBJECT; + return writePath; + } + + // 2. Check processors/ if (io.time().processorCase()) { - fileName objectPath = processorsPath(io, io.instance())/io.name(); - if (isFileOrDir(isFile, objectPath)) + tmpNrc<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath())); + forAll(pDirs(), i) { - searchType = fileOperation::PROCESSORSOBJECT; - return objectPath; + const fileName& pDir = pDirs()[i].first(); + fileName objPath = + processorsPath(io, io.instance(), pDir) + /io.name(); + if (objPath != writePath && isFileOrDir(isFile, objPath)) + { + searchType = pDirs()[i].second().first(); + procsDir = pDir; + return objPath; + } } } { - // 2. Check local - fileName localObjectPath = io.objectPath(); + // 3. Check local + fileName localPath = io.objectPath(); - if (isFileOrDir(isFile, localObjectPath)) + if + ( + localPath != writePath + && isFileOrDir(isFile, localPath) + ) { searchType = fileOperation::OBJECT; - return localObjectPath; + return localPath; } } @@ -153,14 +238,14 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo ) ) { - fileName parentObjectPath = + fileName parentPath = io.rootPath()/io.time().globalCaseName() /io.instance()/io.db().dbDir()/io.local()/io.name(); - if (isFileOrDir(isFile, parentObjectPath)) + if (isFileOrDir(isFile, parentPath)) { searchType = fileOperation::PARENTOBJECT; - return parentObjectPath; + return parentPath; } } @@ -181,23 +266,57 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo instant(io.instance()) ); - if (newInstancePath.size()) + if (newInstancePath.size() && newInstancePath != io.instance()) { // 1. Try processors equivalent - - fileName fName = - processorsPath(io, newInstancePath) - /io.name(); - if (isFileOrDir(isFile, fName)) + tmpNrc<dirIndexList> pDirs + ( + lookupProcessorsPath(io.objectPath()) + ); + forAll(pDirs(), i) { - searchType = fileOperation::PROCESSORSFINDINSTANCE; - return fName; + const fileName& pDir = pDirs()[i].first(); + + fileName fName + ( + processorsPath(io, newInstancePath, pDir) + /io.name() + ); + if (isFileOrDir(isFile, fName)) + { + switch (pDirs()[i].second().first()) + { + case fileOperation::PROCUNCOLLATED: + { + searchType = + fileOperation::PROCUNCOLLATEDINSTANCE; + } + break; + case fileOperation::PROCBASEOBJECT: + { + searchType = fileOperation::PROCBASEINSTANCE; + } + break; + case fileOperation::PROCOBJECT: + { + searchType = fileOperation::PROCINSTANCE; + } + break; + default: + break; + } + procsDir = pDir; + return fName; + } } - fName = - io.rootPath()/io.caseName() - /newInstancePath/io.db().dbDir()/io.local()/io.name(); + // 2. Check local + fileName fName + ( + io.rootPath()/io.caseName() + /newInstancePath/io.db().dbDir()/io.local()/io.name() + ); if (isFileOrDir(isFile, fName)) { searchType = fileOperation::FINDINSTANCE; @@ -213,112 +332,13 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo Foam::fileName -Foam::fileOperations::masterUncollatedFileOperation::processorsCasePath -( - const IOobject& io -) -{ - return - io.rootPath() - /io.time().globalCaseName() - /processorsDir; -} - - -Foam::fileName -Foam::fileOperations::masterUncollatedFileOperation::processorsPath -( - const IOobject& io, - const word& instance -) -{ - return - processorsCasePath(io) - /instance - /io.db().dbDir() - /io.local(); -} - - -Foam::fileName -Foam::fileOperations::masterUncollatedFileOperation::processorsPath -( - const fileName& dir -) -{ - // Check if directory is processorXXX - word caseName(dir.name()); - - std::string::size_type pos = caseName.find("processor"); - if (pos == 0) - { - return dir.path()/processorsDir; - } - else - { - return fileName::null; - } -} - - -Foam::label -Foam::fileOperations::masterUncollatedFileOperation::splitProcessorPath -( - const fileName& objectPath, - fileName& path, - fileName& local -) -{ - // Search for processor at start of line or /processor - std::string::size_type pos = objectPath.find("processor"); - if (pos == string::npos) - { - return -1; - } - - if (pos == 0) - { - path = ""; - local = objectPath.substr(pos+9); - } - else if (objectPath[pos-1] != '/') - { - return -1; - } - else - { - path = objectPath.substr(0, pos-1); - local = objectPath.substr(pos+9); - } - - label proci; - - pos = local.find('/'); - if (pos == string::npos) - { - // processorXXX without local - if (Foam::read(local, proci)) - { - local.clear(); - return proci; - } - } - else if (Foam::read(local.substr(0, pos), proci)) - { - local = local.substr(pos+1); - return proci; - } - - return -1; -} - - -Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::objectPath +Foam::fileOperations::masterUncollatedFileOperation::localObjectPath ( const IOobject& io, const pathType& searchType, + const word& procDir, const word& instancePath -) +) const { // Replacement for IOobject::objectPath() @@ -336,9 +356,50 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::objectPath } break; - case fileOperation::PROCESSORSOBJECT: + case fileOperation::WRITEOBJECT: { - return processorsPath(io, io.instance())/io.name(); + return objectPath(io, io.headerClassName()); + } + break; + + case fileOperation::PROCUNCOLLATED: + { + // Uncollated type, e.g. processor1 + const word procName + ( + "processor" + +Foam::name(Pstream::myProcNo(Pstream::worldComm)) + ); + return + processorsPath + ( + io, + io.instance(), + ( + Pstream::parRun() + ? procName + : procDir + ) + ) + /io.name(); + } + break; + + case fileOperation::PROCBASEOBJECT: + { + // Collated, e.g. processors4 + return + processorsPath(io, io.instance(), procDir) + /io.name(); + } + break; + + case fileOperation::PROCOBJECT: + { + // Processors directory locally provided by the fileHandler itself + return + processorsPath(io, io.instance(), processorsDir(io)) + /io.name(); } break; @@ -358,9 +419,44 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::objectPath } break; - case fileOperation::PROCESSORSFINDINSTANCE: + case fileOperation::PROCUNCOLLATEDINSTANCE: { - return processorsPath(io, instancePath)/io.name(); + // Uncollated type, e.g. processor1 + const word procName + ( + "processor" + +Foam::name(Pstream::myProcNo(Pstream::worldComm)) + ); + return + processorsPath + ( + io, + instancePath, + ( + Pstream::parRun() + ? procName + : procDir + ) + ) + /io.name(); + } + break; + + case fileOperation::PROCBASEINSTANCE: + { + // Collated, e.g. processors4 + return + processorsPath(io, instancePath, procDir) + /io.name(); + } + break; + + case fileOperation::PROCINSTANCE: + { + // Processors directory locally provided by the fileHandler itself + return + processorsPath(io, instancePath, processorsDir(io)) + /io.name(); } break; @@ -409,12 +505,18 @@ void Foam::fileOperations::masterUncollatedFileOperation::readAndSend { if (debug) { - Pout<< "masterUncollatedFileOperation::readAndSend:" - << " opening compressed " << filePath << endl; + Pout<< "masterUncollatedFileOperation::readAndSend :" + << " Opening compressed " << filePath << endl; } IFstream is(filePath, IOstream::streamFormat::BINARY); + if (!is.good()) + { + FatalIOErrorInFunction(filePath) << "Cannot open file " << filePath + << exit(FatalIOError); + } + std::ostringstream stringStr; stringStr << is.stdStream().rdbuf(); string buf(stringStr.str()); @@ -430,9 +532,16 @@ void Foam::fileOperations::masterUncollatedFileOperation::readAndSend off_t count(Foam::fileSize(filePath)); IFstream is(filePath, IOstream::streamFormat::BINARY); + if (!is.good()) + { + FatalIOErrorInFunction(filePath) << "Cannot open file " << filePath + << exit(FatalIOError); + } + + if (debug) { - Pout<< "masterUncollatedFileOperation::readStream:" + Pout<< "masterUncollatedFileOperation::readStream :" << " From " << filePath << " reading " << label(count) << " bytes" << endl; } @@ -448,6 +557,195 @@ void Foam::fileOperations::masterUncollatedFileOperation::readAndSend } +void Foam::fileOperations::masterUncollatedFileOperation::readAndSend +( + const fileName& fName, + const labelUList& procs, + PstreamBuffers& pBufs +) +{ + if (Foam::exists(fName+".gz", false)) + { + readAndSend + ( + fName, + IOstream::compressionType::COMPRESSED, + procs, + pBufs + ); + } + else + { + readAndSend + ( + fName, + IOstream::compressionType::UNCOMPRESSED, + procs, + pBufs + ); + } +} + + +Foam::autoPtr<Foam::ISstream> +Foam::fileOperations::masterUncollatedFileOperation::read +( + IOobject& io, + const label comm, + const bool uniform, // on comms master only + const fileNameList& filePaths, // on comms master only + const boolList& procValid // on comms master only +) +{ + autoPtr<ISstream> isPtr; + + // const bool uniform = uniformFile(filePaths); + + PstreamBuffers pBufs + ( + Pstream::commsTypes::nonBlocking, + Pstream::msgType(), + comm + ); + + if (Pstream::master(comm)) + { + if (uniform) + { + if (procValid[0]) + { + DynamicList<label> validProcs(Pstream::nProcs(comm)); + for + ( + label proci = 0; + proci < Pstream::nProcs(comm); + proci++ + ) + { + if (procValid[proci]) + { + validProcs.append(proci); + } + } + + // Read on master and send to all processors (including + // master for simplicity) + if (debug) + { + Pout<< "masterUncollatedFileOperation::readStream :" + << " For uniform file " << filePaths[0] + << " sending to " << validProcs + << " in comm:" << comm << endl; + } + readAndSend(filePaths[0], validProcs, pBufs); + } + } + else + { + if (procValid[0]) + { + if (filePaths[0].empty()) + { + FatalIOErrorInFunction(filePaths[0]) + << "cannot find file " << io.objectPath() + << exit(FatalIOError); + } + + autoPtr<IFstream> ifsPtr(new IFstream(filePaths[0])); + + // Read header + if (!io.readHeader(ifsPtr())) + { + FatalIOErrorInFunction(ifsPtr()) + << "problem while reading header for object " + << io.name() << exit(FatalIOError); + } + + // Open master (steal from ifsPtr) + isPtr.reset(ifsPtr.ptr()); + } + + // Read slave files + for + ( + label proci = 1; + proci < Pstream::nProcs(comm); + proci++ + ) + { + if (debug) + { + Pout<< "masterUncollatedFileOperation::readStream :" + << " For processor " << proci + << " opening " << filePaths[proci] << endl; + } + + const fileName& fPath = filePaths[proci]; + + if (procValid[proci] && !fPath.empty()) + { + // Note: handle compression ourselves since size cannot + // be determined without actually uncompressing + readAndSend(fPath, labelList(1, proci), pBufs); + } + } + } + } + + labelList recvSizes; + pBufs.finishedSends(recvSizes); + + // isPtr will be valid on master and will be the unbuffered + // IFstream. Else the information is in the PstreamBuffers (and + // the special case of a uniform file) + + if (procValid[Pstream::myProcNo(comm)]) + { + // This processor needs to return something + + if (!isPtr.valid()) + { + UIPstream is(Pstream::masterNo(), pBufs); + string buf(recvSizes[Pstream::masterNo()], '\0'); + if (recvSizes[Pstream::masterNo()] > 0) + { + is.read(&buf[0], recvSizes[Pstream::masterNo()]); + } + + if (debug) + { + Pout<< "masterUncollatedFileOperation::readStream :" + << " Done reading " << buf.size() << " bytes" << endl; + } + const fileName& fName = filePaths[Pstream::myProcNo(comm)]; + isPtr.reset + ( + new IStringStream + ( + buf, + IOstream::ASCII, + IOstream::currentVersion, + fName + ) + ); + + if (!io.readHeader(isPtr())) + { + FatalIOErrorInFunction(isPtr()) + << "problem while reading header for object " + << io.name() << exit(FatalIOError); + } + } + } + else + { + isPtr.reset(new dummyISstream()); + } + + return isPtr; +} + + // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // Foam::fileOperations::masterUncollatedFileOperation:: @@ -455,6 +753,59 @@ masterUncollatedFileOperation ( const bool verbose ) +: + fileOperation + ( + UPstream::allocateCommunicator + ( + UPstream::worldComm, + subRanks(Pstream::nProcs()) + ) + ), + myComm_(comm_) +{ + if (verbose) + { + Info<< "I/O : " << typeName + << " (maxMasterFileBufferSize " << maxMasterFileBufferSize << ')' + << endl; + } + + if (regIOobject::fileModificationChecking == regIOobject::timeStampMaster) + { + if (verbose) + { + WarningInFunction + << "Resetting fileModificationChecking to timeStamp" << endl; + } + regIOobject::fileModificationChecking = regIOobject::timeStamp; + } + else if + ( + regIOobject::fileModificationChecking + == regIOobject::inotifyMaster + ) + { + if (verbose) + { + WarningInFunction + << "Resetting fileModificationChecking to inotify" + << endl; + } + regIOobject::fileModificationChecking = regIOobject::inotify; + } +} + + +Foam::fileOperations::masterUncollatedFileOperation:: +masterUncollatedFileOperation +( + const label comm, + const bool verbose +) +: + fileOperation(comm), + myComm_(-1) { if (verbose) { @@ -489,6 +840,48 @@ masterUncollatedFileOperation } +Foam::fileOperations::masterUncollatedFileOperationInitialise:: +masterUncollatedFileOperationInitialise(int& argc, char**& argv) +: + unthreadedInitialise(argc, argv) +{ + // Filter out any of my arguments + const string s("-ioRanks"); + + int index = -1; + for (int i=1; i<argc-1; i++) + { + if (argv[i] == s) + { + index = i; + setEnv("FOAM_IORANKS", argv[i+1], true); + break; + } + } + + if (index != -1) + { + for (int i=index+2; i<argc; i++) + { + argv[i-2] = argv[i]; + } + argc -= 2; + } +} + + +// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // + +Foam::fileOperations::masterUncollatedFileOperation:: +~masterUncollatedFileOperation() +{ + if (myComm_ != -1 && myComm_ != UPstream::worldComm) + { + UPstream::freeCommunicator(myComm_); + } +} + + // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // bool Foam::fileOperations::masterUncollatedFileOperation::mkDir @@ -497,7 +890,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::mkDir mode_t mode ) const { - return masterOp<mode_t, mkDirOp>(dir, mkDirOp(mode)); + return masterOp<mode_t, mkDirOp> + ( + dir, + mkDirOp(mode), + Pstream::msgType(), + comm_ + ); } @@ -507,7 +906,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::chMod mode_t mode ) const { - return masterOp<mode_t, chModOp>(fName, chModOp(mode)); + return masterOp<mode_t, chModOp> + ( + fName, + chModOp(mode), + Pstream::msgType(), + comm_ + ); } @@ -517,7 +922,13 @@ mode_t Foam::fileOperations::masterUncollatedFileOperation::mode const bool followLink ) const { - return masterOp<mode_t, modeOp>(fName, modeOp(followLink)); + return masterOp<mode_t, modeOp> + ( + fName, + modeOp(followLink), + Pstream::msgType(), + comm_ + ); } @@ -527,7 +938,16 @@ Foam::fileName::Type Foam::fileOperations::masterUncollatedFileOperation::type const bool followLink ) const { - return fileName::Type(masterOp<label, typeOp>(fName, typeOp(followLink))); + return fileName::Type + ( + masterOp<label, typeOp> + ( + fName, + typeOp(followLink), + Pstream::msgType(), + comm_ + ) + ); } @@ -538,7 +958,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::exists const bool followLink ) const { - return masterOp<bool, existsOp>(fName, existsOp(checkGzip, followLink)); + return masterOp<bool, existsOp> + ( + fName, + existsOp(checkGzip, followLink), + Pstream::msgType(), + comm_ + ); } @@ -548,7 +974,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::isDir const bool followLink ) const { - return masterOp<bool, isDirOp>(fName, isDirOp(followLink)); + return masterOp<bool, isDirOp> + ( + fName, + isDirOp(followLink), + Pstream::msgType(), + comm_ + ); } @@ -559,7 +991,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::isFile const bool followLink ) const { - return masterOp<bool, isFileOp>(fName, isFileOp(checkGzip, followLink)); + return masterOp<bool, isFileOp> + ( + fName, + isFileOp(checkGzip, followLink), + Pstream::msgType(), + comm_ + ); } @@ -569,7 +1007,13 @@ off_t Foam::fileOperations::masterUncollatedFileOperation::fileSize const bool followLink ) const { - return masterOp<off_t, fileSizeOp>(fName, fileSizeOp(followLink)); + return masterOp<off_t, fileSizeOp> + ( + fName, + fileSizeOp(followLink), + Pstream::msgType(), + comm_ + ); } @@ -582,7 +1026,9 @@ time_t Foam::fileOperations::masterUncollatedFileOperation::lastModified return masterOp<time_t, lastModifiedOp> ( fName, - lastModifiedOp(followLink) + lastModifiedOp(followLink), + Pstream::msgType(), + comm_ ); } @@ -596,7 +1042,9 @@ double Foam::fileOperations::masterUncollatedFileOperation::highResLastModified return masterOp<double, lastModifiedHROp> ( fName, - lastModifiedHROp(followLink) + lastModifiedHROp(followLink), + Pstream::msgType(), + comm_ ); } @@ -607,7 +1055,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::mvBak const std::string& ext ) const { - return masterOp<bool, mvBakOp>(fName, mvBakOp(ext)); + return masterOp<bool, mvBakOp> + ( + fName, + mvBakOp(ext), + Pstream::msgType(), + comm_ + ); } @@ -616,7 +1070,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::rm const fileName& fName ) const { - return masterOp<bool, rmOp>(fName, rmOp()); + return masterOp<bool, rmOp> + ( + fName, + rmOp(), + Pstream::msgType(), + comm_ + ); } @@ -626,7 +1086,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::rmDir const bool silent ) const { - return masterOp<bool, rmDirOp>(dir, rmDirOp(silent)); + return masterOp<bool, rmDirOp> + ( + dir, + rmDirOp(silent), + Pstream::msgType(), + comm_ + ); } @@ -641,7 +1107,9 @@ Foam::fileNameList Foam::fileOperations::masterUncollatedFileOperation::readDir return masterOp<fileNameList, readDirOp> ( dir, - readDirOp(type, filtergz, followLink) + readDirOp(type, filtergz, followLink), + Pstream::msgType(), + comm_ ); } @@ -653,7 +1121,14 @@ bool Foam::fileOperations::masterUncollatedFileOperation::cp const bool followLink ) const { - return masterOp<bool, cpOp>(src, dst, cpOp(followLink)); + return masterOp<bool, cpOp> + ( + src, + dst, + cpOp(followLink), + Pstream::msgType(), + comm_ + ); } @@ -663,7 +1138,14 @@ bool Foam::fileOperations::masterUncollatedFileOperation::ln const fileName& dst ) const { - return masterOp<bool, lnOp>(src, dst, lnOp()); + return masterOp<bool, lnOp> + ( + src, + dst, + lnOp(), + Pstream::msgType(), + comm_ + ); } @@ -674,7 +1156,14 @@ bool Foam::fileOperations::masterUncollatedFileOperation::mv const bool followLink ) const { - return masterOp<bool, mvOp>(src, dst, mvOp(followLink)); + return masterOp<bool, mvOp> + ( + src, + dst, + mvOp(followLink), + Pstream::msgType(), + comm_ + ); } @@ -693,17 +1182,25 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath << " checkGlobal:" << checkGlobal << endl; } + // Now that we have an IOobject path use it to detect & cache + // processor directory naming + (void)lookupProcessorsPath(io.objectPath()); + // Trigger caching of times (void)findTimes(io.time().path(), io.time().constant()); + // Determine master filePath and scatter fileName objPath; pathType searchType = NOTFOUND; + word procsDir; word newInstancePath; - if (Pstream::master()) + if (Pstream::master(comm_)) { + // All masters search locally. Note that global objects might + // fail (except on master). This gets handled later on (in PARENTOBJECT) objPath = filePathInfo ( @@ -712,137 +1209,477 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath io, search, searchType, + procsDir, newInstancePath ); + + if (debug) + { + Pout<< "masterUncollatedFileOperation::filePath :" + << " master objPath:" << objPath + << " searchType:" << fileOperation::pathTypeNames_[searchType] + << " procsDir:" << procsDir << " instance:" << newInstancePath + << endl; + } } + // Scatter the information about where the master found the object + // Note: use the worldComm to make sure all processors decide + // the same type. Only procsDir is allowed to differ; searchType + // and instance have to be same { label masterType(searchType); Pstream::scatter(masterType); searchType = pathType(masterType); } - Pstream::scatter(newInstancePath); - - // Use the master type to determine if additional information is - // needed to construct the local equivalent - switch (searchType) + if + ( + checkGlobal + || searchType == fileOperation::PARENTOBJECT + || searchType == fileOperation::PROCBASEOBJECT + || searchType == fileOperation::PROCBASEINSTANCE + || io.local() == "uniform" + ) { - case fileOperation::ABSOLUTE: - case fileOperation::PROCESSORSOBJECT: - case fileOperation::PARENTOBJECT: - case fileOperation::FINDINSTANCE: - case fileOperation::PROCESSORSFINDINSTANCE: + // Distribute master path. This makes sure it is seen as uniform + // and only gets read from the master. + Pstream::scatter(objPath); + Pstream::scatter(procsDir); + } + else + { + Pstream::scatter(procsDir, Pstream::msgType(), comm_); + + // Use the master type to determine if additional information is + // needed to construct the local equivalent + switch (searchType) { - // Construct equivalent local path - objPath = objectPath(io, searchType, newInstancePath); + case fileOperation::PARENTOBJECT: + case fileOperation::PROCBASEOBJECT: + case fileOperation::PROCBASEINSTANCE: + { + // Already handled above + } + break; + + case fileOperation::ABSOLUTE: + case fileOperation::WRITEOBJECT: + case fileOperation::PROCUNCOLLATED: + case fileOperation::PROCOBJECT: + case fileOperation::FINDINSTANCE: + case fileOperation::PROCUNCOLLATEDINSTANCE: + case fileOperation::PROCINSTANCE: + { + // Construct equivalent local path + objPath = localObjectPath + ( + io, + searchType, + procsDir, + newInstancePath + ); + } + break; + + case fileOperation::OBJECT: + case fileOperation::NOTFOUND: + { + // Retest all processors separately since some processors might + // have the file and some not (e.g. lagrangian data) + + objPath = masterOp<fileName, fileOrNullOp> + ( + io.objectPath(), + fileOrNullOp(true), + Pstream::msgType(), + comm_ + ); + } + break; } - break; + } - case fileOperation::OBJECT: - case fileOperation::NOTFOUND: + if (debug) + { + Pout<< "masterUncollatedFileOperation::filePath :" + << " Returning from file searching:" << endl + << " objectPath:" << io.objectPath() << endl + << " filePath :" << objPath << endl << endl; + } + return objPath; +} + + +Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath +( + const bool checkGlobal, + const IOobject& io, + const bool search +) const +{ + if (debug) + { + Pout<< "masterUncollatedFileOperation::dirPath :" + << " objectPath:" << io.objectPath() + << " checkGlobal:" << checkGlobal << endl; + } + + // Now that we have an IOobject path use it to detect & cache + // processor directory naming + (void)lookupProcessorsPath(io.objectPath()); + + // Determine master dirPath and scatter + + fileName objPath; + pathType searchType = NOTFOUND; + word procsDir; + word newInstancePath; + + if (Pstream::master(comm_)) + { + objPath = filePathInfo + ( + checkGlobal, + false, + io, + search, + searchType, + procsDir, + newInstancePath + ); + } + + { + label masterType(searchType); + Pstream::scatter(masterType); //, Pstream::msgType(), comm_); + searchType = pathType(masterType); + } + Pstream::scatter(newInstancePath); //, Pstream::msgType(), comm_); + + if + ( + checkGlobal + || searchType == fileOperation::PARENTOBJECT + || searchType == fileOperation::PROCBASEOBJECT + || searchType == fileOperation::PROCBASEINSTANCE + || io.local() == "uniform" + ) + { + // Distribute master path. This makes sure it is seen as uniform + // and only gets read from the master. + Pstream::scatter(objPath); + Pstream::scatter(procsDir); + } + else + { + Pstream::scatter(procsDir, Pstream::msgType(), comm_); + + // Use the master type to determine if additional information is + // needed to construct the local equivalent + switch (searchType) { - // Retest all processors separately since some processors might - // have the file and some not (e.g. lagrangian data) - objPath = masterOp<fileName, fileOrNullOp> - ( - io.objectPath(), - fileOrNullOp(true) - ); + case fileOperation::PARENTOBJECT: + case fileOperation::PROCBASEOBJECT: + case fileOperation::PROCBASEINSTANCE: + { + // Already handled above + } + break; + + case fileOperation::ABSOLUTE: + case fileOperation::WRITEOBJECT: + case fileOperation::PROCUNCOLLATED: + case fileOperation::PROCOBJECT: + case fileOperation::FINDINSTANCE: + case fileOperation::PROCUNCOLLATEDINSTANCE: + case fileOperation::PROCINSTANCE: + { + // Construct equivalent local path + objPath = localObjectPath + ( + io, + searchType, + procsDir, + newInstancePath + ); + } + break; + + case fileOperation::OBJECT: + case fileOperation::NOTFOUND: + { + // Retest all processors separately since some processors might + // have the file and some not (e.g. lagrangian data) + objPath = masterOp<fileName, fileOrNullOp> + ( + io.objectPath(), + fileOrNullOp(false), + Pstream::msgType(), + comm_ + ); + } + break; } - break; } if (debug) { - Pout<< "masterUncollatedFileOperation::filePath :" + Pout<< "masterUncollatedFileOperation::dirPath :" << " Returning from file searching:" << endl << " objectPath:" << io.objectPath() << endl << " filePath :" << objPath << endl << endl; } - return objPath; + return objPath; +} + + +bool Foam::fileOperations::masterUncollatedFileOperation::exists +( + const dirIndexList& pDirs, + IOobject& io +) const +{ + // Cut-down version of filePathInfo that does not look for + // different instance or parent directory + + const bool isFile = !io.name().empty(); + + // Generate output filename for object + const fileName writePath(objectPath(io, word::null)); + + // 1. Test writing name for either directory or a (valid) file + if (isFileOrDir(isFile, writePath)) + { + return true; + } + + // 2. Check processors/ + if (io.time().processorCase()) + { + forAll(pDirs, i) + { + const fileName& pDir = pDirs[i].first(); + fileName procPath = + processorsPath(io, io.instance(), pDir) + /io.name(); + if (procPath != writePath && isFileOrDir(isFile, procPath)) + { + return true; + } + } + } + + // 3. Check local + fileName localPath = io.objectPath(); + + if (localPath != writePath && isFileOrDir(isFile, localPath)) + { + return true; + } + + return false; } -Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath +Foam::IOobject +Foam::fileOperations::masterUncollatedFileOperation::findInstance ( - const bool checkGlobal, - const IOobject& io, - const bool search + const IOobject& startIO, + const scalar startValue, + const word& stopInstance ) const { if (debug) { - Pout<< "masterUncollatedFileOperation::dirPath :" - << " objectPath:" << io.objectPath() - << " checkGlobal:" << checkGlobal << endl; + Pout<< "masterUncollatedFileOperation::findInstance :" + << " Starting searching for name:" << startIO.name() + << " local:" << startIO.local() + << " from instance:" << startIO.instance() + << endl; } - // Determine master dirPath and scatter - fileName objPath; - pathType searchType = NOTFOUND; - word newInstancePath; + const Time& time = startIO.time(); + + IOobject io(startIO); + + // Note: - if name is empty, just check the directory itself + // - check both for isFile and headerOk since the latter does a + // filePath so searches for the file. + // - check for an object with local file scope (so no looking up in + // parent directory in case of parallel) + - if (Pstream::master()) + tmpNrc<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath())); + + word foundInstance; + + // if (Pstream::master(comm_)) + if (Pstream::master(UPstream::worldComm)) { - objPath = - filePathInfo - ( - checkGlobal, - false, - io, - search, - searchType, - newInstancePath - ); + if (exists(pDirs, io)) + { + foundInstance = io.instance(); + } } + + // Do parallel early exit to avoid calling time.times() + // Pstream::scatter(foundInstance, Pstream::msgType(), comm_); + Pstream::scatter(foundInstance, Pstream::msgType(), UPstream::worldComm); + if (!foundInstance.empty()) { - label masterType(searchType); - Pstream::scatter(masterType); - searchType = pathType(masterType); + io.instance() = foundInstance; + if (debug) + { + Pout<< "masterUncollatedFileOperation::findInstance :" + << " for name:" << io.name() << " local:" << io.local() + << " found starting instance:" << io.instance() << endl; + } + return io; } - Pstream::scatter(newInstancePath); - // Use the master type to determine if additional information is - // needed to construct the local equivalent - switch (searchType) + // Search back through the time directories to find the time + // closest to and lower than current time + + instantList ts = time.times(); + // if (Pstream::master(comm_)) + if (Pstream::master(UPstream::worldComm)) { - case fileOperation::ABSOLUTE: - case fileOperation::PROCESSORSOBJECT: - case fileOperation::PARENTOBJECT: - case fileOperation::FINDINSTANCE: - case fileOperation::PROCESSORSFINDINSTANCE: + label instanceI; + + for (instanceI = ts.size()-1; instanceI >= 0; --instanceI) { - // Construct equivalent local path - objPath = objectPath(io, searchType, newInstancePath); + if (ts[instanceI].value() <= startValue) + { + break; + } } - break; - case fileOperation::OBJECT: - case fileOperation::NOTFOUND: + // continue searching from here + for (; instanceI >= 0; --instanceI) + { + // Shortcut: if actual directory is the timeName we've + // already tested it + if (ts[instanceI].name() == time.timeName()) + { + continue; + } + + io.instance() = ts[instanceI].name(); + if (exists(pDirs, io)) + { + foundInstance = io.instance(); + if (debug) + { + Pout<< "masterUncollatedFileOperation::findInstance :" + << " for name:" << io.name() << " local:" << io.local() + << " found at:" << io.instance() + << endl; + } + break; + } + + // Check if hit minimum instance + if (ts[instanceI].name() == stopInstance) + { + if + ( + startIO.readOpt() == IOobject::MUST_READ + || startIO.readOpt() == IOobject::MUST_READ_IF_MODIFIED + ) + { + if (io.name().empty()) + { + FatalErrorInFunction + << "Cannot find directory " + << io.local() << " in times " << time.timeName() + << " down to " << stopInstance + << exit(FatalError); + } + else + { + FatalErrorInFunction + << "Cannot find file \"" << io.name() + << "\" in directory " << io.local() + << " in times " << time.timeName() + << " down to " << stopInstance + << exit(FatalError); + } + } + foundInstance = io.instance(); + if (debug) + { + Pout<< "masterUncollatedFileOperation::findInstance :" + << " name:" << io.name() << " local:" << io.local() + << " found at stopinstance:" << io.instance() << endl; + } + break; + } + } + + + if (foundInstance.empty()) + { + // times() usually already includes the constant() so would + // have been checked above. Re-test if + // - times() is empty. Sometimes this can happen (e.g. decomposePar + // with collated) + // - times()[0] is not constant + if (!ts.size() || ts[0].name() != time.constant()) + { + // Note. This needs to be a hard-coded constant, rather than the + // constant function of the time, because the latter points to + // the case constant directory in parallel cases + + io.instance() = time.constant(); + if (exists(pDirs, io)) + { + if (debug) + { + Pout<< "masterUncollatedFileOperation::findInstance :" + << " name:" << io.name() + << " local:" << io.local() + << " found at:" << io.instance() << endl; + } + foundInstance = io.instance(); + } + } + } + + if (foundInstance.empty()) { - // Retest all processors separately since some processors might - // have the file and some not (e.g. lagrangian data) - objPath = masterOp<fileName, fileOrNullOp> + if ( - io.objectPath(), - fileOrNullOp(false) - ); + startIO.readOpt() == IOobject::MUST_READ + || startIO.readOpt() == IOobject::MUST_READ_IF_MODIFIED + ) + { + FatalErrorInFunction + << "Cannot find file \"" << io.name() << "\" in directory " + << io.local() << " in times " << startIO.instance() + << " down to " << time.constant() + << exit(FatalError); + } + else + { + foundInstance = time.constant(); + } } - break; } + // Pstream::scatter(foundInstance, Pstream::msgType(), comm_); + Pstream::scatter(foundInstance, Pstream::msgType(), UPstream::worldComm); + io.instance() = foundInstance; if (debug) { - Pout<< "masterUncollatedFileOperation::dirPath :" - << " Returning from file searching:" << endl - << " objectPath:" << io.objectPath() << endl - << " filePath :" << objPath << endl << endl; + Pout<< "masterUncollatedFileOperation::findInstance :" + << " name:" << io.name() << " local:" << io.local() + << " returning instance:" << io.instance() << endl; } - return objPath; + return io; } @@ -859,14 +1696,22 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects { Pout<< "masterUncollatedFileOperation::readObjects :" << " db:" << db.objectPath() - << " instance:" << instance << endl; + << " local:" << local << " instance:" << instance << endl; } fileNameList objectNames; newInstance = word::null; - if (Pstream::master()) + // Note: readObjects uses WORLD to make sure order of objects is the + // same everywhere + + if (Pstream::master()) // comm_)) { + // Avoid fileOperation::readObjects from triggering parallel ops + // (through call to filePath which triggers parallel ) + bool oldParRun = UPstream::parRun(); + UPstream::parRun() = false; + //- Use non-time searching version objectNames = fileOperation::readObjects ( @@ -908,10 +1753,12 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects } } } + + UPstream::parRun() = oldParRun; } - Pstream::scatter(newInstance); - Pstream::scatter(objectNames); + Pstream::scatter(newInstance); //, Pstream::msgType(), comm_); + Pstream::scatter(objectNames); //, Pstream::msgType(), comm_); if (debug) { @@ -935,21 +1782,21 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader if (debug) { - Pout<< "masterUncollatedFileOperation::readHeader:" << endl + Pout<< "masterUncollatedFileOperation::readHeader :" << endl << " objectPath:" << io.objectPath() << endl << " fName :" << fName << endl; } - fileNameList filePaths(Pstream::nProcs()); - filePaths[Pstream::myProcNo()] = fName; - Pstream::gatherList(filePaths); - + // Get filePaths on world master + fileNameList filePaths(Pstream::nProcs(Pstream::worldComm)); + filePaths[Pstream::myProcNo(Pstream::worldComm)] = fName; + Pstream::gatherList(filePaths, Pstream::msgType(), Pstream::worldComm); bool uniform = uniformFile(filePaths); - Pstream::scatter(uniform); + Pstream::scatter(uniform, Pstream::msgType(), Pstream::worldComm); if (uniform) { - if (Pstream::master()) + if (Pstream::master(Pstream::worldComm)) { if (!fName.empty()) { @@ -958,7 +1805,6 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader if (is.good()) { ok = io.readHeader(is); - if (io.headerClassName() == decomposedBlockData::typeName) { // Read the header inside the container (master data) @@ -967,55 +1813,82 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader } } } - Pstream::scatter(ok); - Pstream::scatter(io.headerClassName()); - Pstream::scatter(io.note()); + Pstream::scatter(ok, Pstream::msgType(), Pstream::worldComm); + Pstream::scatter + ( + io.headerClassName(), + Pstream::msgType(), + Pstream::worldComm + ); + Pstream::scatter(io.note(), Pstream::msgType(), Pstream::worldComm); } else { - boolList result(Pstream::nProcs(), false); - wordList headerClassName(Pstream::nProcs()); - stringList note(Pstream::nProcs()); - if (Pstream::master()) + if (Pstream::nProcs(comm_) != Pstream::nProcs(Pstream::worldComm)) + { + // Re-gather file paths on local master + filePaths.setSize(Pstream::nProcs(comm_)); + filePaths[Pstream::myProcNo(comm_)] = fName; + Pstream::gatherList(filePaths, Pstream::msgType(), comm_); + } + + boolList result(Pstream::nProcs(comm_), false); + wordList headerClassName(Pstream::nProcs(comm_)); + stringList note(Pstream::nProcs(comm_)); + if (Pstream::master(comm_)) { forAll(filePaths, proci) { if (!filePaths[proci].empty()) { - IFstream is(filePaths[proci]); - - if (is.good()) + if (proci > 0 && filePaths[proci] == filePaths[proci-1]) + { + result[proci] = result[proci-1]; + headerClassName[proci] = headerClassName[proci-1]; + note[proci] = note[proci-1]; + } + else { - result[proci] = io.readHeader(is); - headerClassName[proci] = io.headerClassName(); - note[proci] = io.note(); - - if - ( - io.headerClassName() - == decomposedBlockData::typeName - ) + IFstream is(filePaths[proci]); + + if (is.good()) { - FatalErrorInFunction - << "Unexpected decomposedBlockData container" - << " for processor " << proci - << " file:" << filePaths[proci] - << ". A decomposedBlockData container should" - << " produce the same file name on all" - << " processors" << exit(FatalError); + result[proci] = io.readHeader(is); + if + ( + io.headerClassName() + == decomposedBlockData::typeName + ) + { + // Read the header inside the container (master + // data) + result[proci] = decomposedBlockData:: + readMasterHeader + ( + io, + is + ); + } + headerClassName[proci] = io.headerClassName(); + note[proci] = io.note(); } } } } } - ok = scatterList(result); - io.headerClassName() = scatterList(headerClassName); - io.note() = scatterList(note); + ok = scatterList(result, Pstream::msgType(), comm_); + io.headerClassName() = scatterList + ( + headerClassName, + Pstream::msgType(), + comm_ + ); + io.note() = scatterList(note, Pstream::msgType(), comm_); } if (debug) { - Pout<< "masterUncollatedFileOperation::readHeader:" << " ok:" << ok + Pout<< "masterUncollatedFileOperation::readHeader :" << " ok:" << ok << " class:" << io.headerClassName() << endl; } return ok; @@ -1033,15 +1906,20 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream { if (debug) { - Pout<< "masterUncollatedFileOperation::readStream:" + Pout<< "masterUncollatedFileOperation::readStream :" << " object : " << io.name() + << " global : " << io.global() << " fName : " << fName << " valid:" << valid << endl; } autoPtr<ISstream> isPtr; bool isCollated = false; - if (UPstream::master()) + IOobject headerIO(io); + + // Detect collated format. This could be done on the local communicator + // but we do it on the master node only for now. + if (UPstream::master()) // comm_)) { if (!fName.empty()) { @@ -1056,13 +1934,26 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream if (isPtr().good()) { // Read header data (on copy) - IOobject headerIO(io); headerIO.readHeader(isPtr()); if (headerIO.headerClassName() == decomposedBlockData::typeName) { isCollated = true; } + else if (!Pstream::parRun()) + { + // Short circuit: non-collated format. No parallel bits. + // Copy header and return. + if (debug) + { + Pout<< "masterUncollatedFileOperation::readStream :" + << " For object : " << io.name() + << " doing straight IFstream input from " + << fName << endl; + } + io = headerIO; + return isPtr; + } } if (!isCollated) @@ -1073,31 +1964,39 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream } } - Pstream::scatter(isCollated); + Pstream::scatter(isCollated); //, Pstream::msgType(), comm_); if (isCollated) { if (debug) { - Pout<< "masterUncollatedFileOperation::readStream:" - << " for object : " << io.name() + Pout<< "masterUncollatedFileOperation::readStream :" + << " For object : " << io.name() << " starting collating input from " << fName << endl; } + + // Analyse the file path (on (co)master) to see the processors type + fileName path, procDir, local; + label groupStart, groupSize, nProcs; + splitProcessorPath + ( + fName, + path, + procDir, + local, + groupStart, + groupSize, + nProcs + ); + + List<char> data; if (!Pstream::parRun()) { // Analyse the objectpath to find out the processor we're trying // to access - fileName path; - fileName local; - label proci = fileOperations::masterUncollatedFileOperation:: - splitProcessorPath - ( - io.objectPath(), - path, - local - ); + label proci = detectProcessorPath(io.objectPath()); if (proci == -1) { @@ -1107,22 +2006,80 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream << exit(FatalIOError); } + // Analyse the fileName for any processor subset. Note: this + // should really be part of filePath() which should return + // both file and index in file. + if (groupStart != -1 && groupSize > 0) + { + proci = proci-groupStart; + } + + if (debug) + { + Pout<< "masterUncollatedFileOperation::readStream :" + << " For object : " << io.name() + << " starting input from block " << proci + << " of " << isPtr().name() << endl; + } + return decomposedBlockData::readBlock(proci, isPtr(), io); } else { - // Get size of file (on master, scatter to slaves) - off_t sz = fileSize(fName); + // Scatter master header info + string versionString; + string formatString; + if (isPtr.valid()) + { + versionString = isPtr().version().str(); + OStringStream os; + os << isPtr().format(); + formatString = (os.str()); + } + + Pstream::scatter(versionString); //, Pstream::msgType(), comm); + Pstream::scatter(formatString); //, Pstream::msgType(), comm); + + // Get size of file + off_t sz = Foam::fileSize(fName); + bool bigSize = sz > off_t(maxMasterFileBufferSize); + Pstream::scatter(bigSize); + + // Are we reading from single-master file ('processors256') or + // from multi-master files ('processors256_0-9') + label readComm = -1; + if (groupStart != -1 && groupSize > 0) + { + readComm = comm_; + if (UPstream::master(comm_) && !isPtr.valid() && !fName.empty()) + { + // In multi-master mode also open the file on the other + // masters + isPtr.reset(new IFstream(fName)); + + if (isPtr().good()) + { + // Read header data (on copy) + IOobject headerIO(io); + headerIO.readHeader(isPtr()); + } + } + } + else + { + // Single master so read on world + readComm = Pstream::worldComm; + } // Read my data return decomposedBlockData::readBlocks ( - UPstream::worldComm, + readComm, fName, isPtr, io, ( - sz > off_t(maxMasterFileBufferSize) + bigSize ? UPstream::commsTypes::scheduled : UPstream::commsTypes::nonBlocking ) @@ -1133,145 +2090,38 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream { if (debug) { - Pout<< "masterUncollatedFileOperation::readStream:" - << " for object : " << io.name() + Pout<< "masterUncollatedFileOperation::readStream :" + << " For object : " << io.name() << " starting separated input from " << fName << endl; } - fileNameList filePaths(Pstream::nProcs()); - filePaths[Pstream::myProcNo()] = fName; - Pstream::gatherList(filePaths); - boolList procValid(Pstream::nProcs()); - procValid[Pstream::myProcNo()] = valid; - Pstream::gatherList(procValid); - - PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking); - - if (Pstream::master()) - { - //const bool uniform = uniformFile(filePaths); - - if (valid) - { - if (fName.empty()) - { - FatalErrorInFunction - << "cannot find file " << io.objectPath() - << exit(FatalError); - } - else - { - autoPtr<IFstream> ifsPtr(new IFstream(fName)); - - // Read header - if (!io.readHeader(ifsPtr())) - { - FatalIOErrorInFunction(ifsPtr()) - << "problem while reading header for object " - << io.name() << exit(FatalIOError); - } - - // Open master (steal from ifsPtr) - isPtr.reset(ifsPtr.ptr()); - } - } - - // Read slave files - for (label proci = 1; proci < Pstream::nProcs(); proci++) - { - if (debug) - { - Pout<< "masterUncollatedFileOperation::readStream:" - << " For processor " << proci - << " opening " << filePaths[proci] << endl; - } - - if (procValid[proci] && !filePaths[proci].empty()) - { - // Note: handle compression ourselves since size cannot - // be determined without actually uncompressing - - if (Foam::exists(filePaths[proci]+".gz", false)) - { - readAndSend - ( - filePaths[proci], - IOstream::compressionType::COMPRESSED, - labelList(1, proci), - pBufs - ); - } - else - { - readAndSend - ( - filePaths[proci], - IOstream::compressionType::UNCOMPRESSED, - labelList(1, proci), - pBufs - ); - } - } - } - } - - labelList recvSizes; - pBufs.finishedSends(recvSizes); - - // isPtr will be valid on master. Else the information is in the - // PstreamBuffers - - if (Pstream::master()) + if (io.global()) { - if (!isPtr.valid()) - { - return autoPtr<ISstream>(new dummyISstream()); - } - else - { - return isPtr; - } + // Use worldComm. Note: should not really need to gather filePaths + // since we enforce sending from master anyway ... + fileNameList filePaths(Pstream::nProcs()); + filePaths[Pstream::myProcNo()] = fName; + Pstream::gatherList(filePaths); + boolList procValid(Pstream::nProcs()); + procValid[Pstream::myProcNo()] = valid; + Pstream::gatherList(procValid); + + return read(io, Pstream::worldComm, true, filePaths, procValid); } else { - if (valid) - { - UIPstream is(Pstream::masterNo(), pBufs); - string buf(recvSizes[Pstream::masterNo()], '\0'); - if (recvSizes[Pstream::masterNo()] > 0) - { - is.read(&buf[0], recvSizes[Pstream::masterNo()]); - } - - if (debug) - { - Pout<< "masterUncollatedFileOperation::readStream:" - << " Done reading " << buf.size() << " bytes" << endl; - } - isPtr.reset - ( - new IStringStream - ( - buf, - IOstream::ASCII, - IOstream::currentVersion, - fName - ) - ); - - if (!io.readHeader(isPtr())) - { - FatalIOErrorInFunction(isPtr()) - << "problem while reading header for object " - << io.name() << exit(FatalIOError); - } - - return isPtr; - } - else - { - return autoPtr<ISstream>(new dummyISstream()); - } + // Use local communicator + fileNameList filePaths(Pstream::nProcs(comm_)); + filePaths[Pstream::myProcNo(comm_)] = fName; + Pstream::gatherList(filePaths, Pstream::msgType(), comm_); + boolList procValid(Pstream::nProcs(comm_)); + procValid[Pstream::myProcNo(comm_)] = valid; + Pstream::gatherList(procValid, Pstream::msgType(), comm_); + + // Uniform in local comm + bool uniform = uniformFile(filePaths); + + return read(io, comm_, uniform, filePaths, procValid); } } } @@ -1291,12 +2141,12 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read { if (debug) { - Pout<< "masterUncollatedFileOperation::read:" - << "reading global object " << io.name() << endl; + Pout<< "masterUncollatedFileOperation::read :" + << " Reading global object " << io.name() << endl; } bool ok = false; - if (Pstream::master()) + if (Pstream::master()) // comm_)) { // Do master-only reading always. bool oldParRun = UPstream::parRun(); @@ -1308,21 +2158,29 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read UPstream::parRun() = oldParRun; } - Pstream::scatter(ok); - Pstream::scatter(io.headerClassName()); - Pstream::scatter(io.note()); + Pstream::scatter(ok); //, Pstream::msgType(), comm_); + Pstream::scatter(io.headerClassName()); //, Pstream::msgType(), comm_); + Pstream::scatter(io.note()); //, Pstream::msgType(), comm_); // scatter operation for regIOobjects // Get my communication order + // const List<Pstream::commsStruct>& comms = + //( + // (Pstream::nProcs(comm_) < Pstream::nProcsSimpleSum) + // ? Pstream::linearCommunication(comm_) + // : Pstream::treeCommunication(comm_) + //); + // const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm_)]; const List<Pstream::commsStruct>& comms = ( - (Pstream::nProcs() < Pstream::nProcsSimpleSum) - ? Pstream::linearCommunication() - : Pstream::treeCommunication() + (Pstream::nProcs(Pstream::worldComm) < Pstream::nProcsSimpleSum) + ? Pstream::linearCommunication(Pstream::worldComm) + : Pstream::treeCommunication(Pstream::worldComm) ); - const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()]; + const Pstream::commsStruct& myComm = + comms[Pstream::myProcNo(Pstream::worldComm)]; // Receive from up if (myComm.above() != -1) @@ -1333,7 +2191,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read myComm.above(), 0, Pstream::msgType(), - Pstream::worldComm, + Pstream::worldComm, // comm_, format ); ok = io.readData(fromAbove); @@ -1348,7 +2206,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read myComm.below()[belowI], 0, Pstream::msgType(), - Pstream::worldComm, + Pstream::worldComm, // comm_, format ); bool okWrite = io.writeData(toBelow); @@ -1359,8 +2217,8 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read { if (debug) { - Pout<< "masterUncollatedFileOperation::read:" - << "reading local object " << io.name() << endl; + Pout<< "masterUncollatedFileOperation::read :" + << " Reading local object " << io.name() << endl; } ok = io.readData(io.readStream(typeName)); @@ -1384,7 +2242,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::writeObject if (debug) { - Pout<< "masterUncollatedFileOperation::writeObject:" + Pout<< "masterUncollatedFileOperation::writeObject :" << " io:" << pathName << " valid:" << valid << endl; } @@ -1433,30 +2291,31 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes const word& constantName ) const { - if (debug) - { - Pout<< "masterUncollatedFileOperation::findTimes:" - << " Finding times in directory " << directory << endl; - } - HashPtrTable<instantList>::const_iterator iter = times_.find(directory); if (iter != times_.end()) { if (debug) { - Pout<< "masterUncollatedFileOperation::findTimes:" - << " Found cached times:" << *iter() << endl; + Pout<< "masterUncollatedFileOperation::findTimes :" + << " Found " << iter()->size() << " cached times" << endl; } return *iter(); } else { instantList times; - if (Pstream::master()) + if (Pstream::master()) // comm_)) { + // Do master-only reading always. + bool oldParRun = UPstream::parRun(); + UPstream::parRun() = false; times = fileOperation::findTimes(directory, constantName); + UPstream::parRun() = oldParRun; } - Pstream::scatter(times); + Pstream::scatter(times); //, Pstream::msgType(), comm_); + + // Note: do we also cache if no times have been found since it might + // indicate a directory that is being filled later on ... instantList* tPtr = new instantList(std::move(times)); @@ -1464,8 +2323,9 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes if (debug) { - Pout<< "masterUncollatedFileOperation::findTimes:" - << " Caching times:" << *tPtr << endl; + Pout<< "masterUncollatedFileOperation::findTimes :" + << " Caching times:" << *tPtr << nl + << " for directory:" << directory << endl; } return *tPtr; } @@ -1477,17 +2337,16 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime const Time& tm ) const { + if (tm.subCycling()) + { + return; + } + HashPtrTable<instantList>::const_iterator iter = times_.find(tm.path()); if (iter != times_.end()) { - if (debug) - { - Pout<< "masterUncollatedFileOperation::setTime:" - << " Caching time " << tm.timeName() - << " for case:" << tm.path() << endl; - } - instantList& times = *iter(); + const instant timeNow(tm.value(), tm.timeName()); if (times.size() > 0 && times[0].name() == tm.constant()) @@ -1504,6 +2363,13 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime == -1 ) { + if (debug) + { + Pout<< "masterUncollatedFileOperation::setTime :" + << " Caching time " << tm.timeName() + << " for case:" << tm.path() << endl; + } + times.append(timeNow); SubList<instant> realTimes(times, times.size()-1, 1); Foam::stableSort(realTimes); @@ -1513,6 +2379,13 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime { if (findSortedIndex(times, timeNow) == -1) { + if (debug) + { + Pout<< "masterUncollatedFileOperation::setTime :" + << " Caching time " << tm.timeName() + << " for case:" << tm.path() << endl; + } + times.append(timeNow); Foam::stableSort(times); } @@ -1533,13 +2406,18 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream // Insert logic of filePath. We assume that if a file is absolute // on the master it is absolute also on the slaves etc. - fileNameList filePaths(Pstream::nProcs()); - filePaths[Pstream::myProcNo()] = filePath; - Pstream::gatherList(filePaths); + fileNameList filePaths(Pstream::nProcs(Pstream::worldComm)); + filePaths[Pstream::myProcNo(Pstream::worldComm)] = filePath; + Pstream::gatherList(filePaths, Pstream::msgType(), Pstream::worldComm); - PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking); + PstreamBuffers pBufs + ( + Pstream::commsTypes::nonBlocking, + Pstream::msgType(), + Pstream::worldComm + ); - if (Pstream::master()) + if (Pstream::master(Pstream::worldComm)) { const bool uniform = uniformFile(filePaths); @@ -1547,7 +2425,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream { if (debug) { - Pout<< "masterUncollatedFileOperation::NewIFstream:" + Pout<< "masterUncollatedFileOperation::NewIFstream :" << " Opening global file " << filePath << endl; } @@ -1558,8 +2436,13 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream : IOstream::compressionType::UNCOMPRESSED ); - labelList procs(Pstream::nProcs()-1); - for (label proci = 1; proci < Pstream::nProcs(); proci++) + labelList procs(Pstream::nProcs(Pstream::worldComm)-1); + for + ( + label proci = 1; + proci < Pstream::nProcs(Pstream::worldComm); + proci++ + ) { procs[proci-1] = proci; } @@ -1568,7 +2451,12 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream } else { - for (label proci = 1; proci < Pstream::nProcs(); proci++) + for + ( + label proci = 1; + proci < Pstream::nProcs(Pstream::worldComm); + proci++ + ) { IOstream::compressionType cmp ( @@ -1592,7 +2480,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream labelList recvSizes; pBufs.finishedSends(recvSizes); - if (Pstream::master()) + if (Pstream::master(Pstream::worldComm)) { // Read myself return autoPtr<ISstream> @@ -1604,7 +2492,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream { if (debug) { - Pout<< "masterUncollatedFileOperation::NewIFstream:" + Pout<< "masterUncollatedFileOperation::NewIFstream :" << " Reading " << filePath << " from processor " << Pstream::masterNo() << endl; } @@ -1615,7 +2503,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream if (debug) { - Pout<< "masterUncollatedFileOperation::NewIFstream:" + Pout<< "masterUncollatedFileOperation::NewIFstream :" << " Done reading " << buf.size() << " bytes" << endl; } @@ -1673,11 +2561,11 @@ Foam::label Foam::fileOperations::masterUncollatedFileOperation::addWatch ) const { label watchFd; - if (Pstream::master()) + if (Pstream::master()) // comm_)) { watchFd = monitor().addWatch(fName); } - Pstream::scatter(watchFd); + Pstream::scatter(watchFd); //, Pstream::msgType(), comm_); return watchFd; } @@ -1688,11 +2576,11 @@ bool Foam::fileOperations::masterUncollatedFileOperation::removeWatch ) const { bool ok; - if (Pstream::master()) + if (Pstream::master()) // comm_)) { ok = monitor().removeWatch(watchIndex); } - Pstream::scatter(ok); + Pstream::scatter(ok); //, Pstream::msgType(), comm_); return ok; } @@ -1705,7 +2593,7 @@ Foam::label Foam::fileOperations::masterUncollatedFileOperation::findWatch { label index = -1; - if (Pstream::master()) + if (Pstream::master()) // comm_)) { forAll(watchIndices, i) { @@ -1716,7 +2604,7 @@ Foam::label Foam::fileOperations::masterUncollatedFileOperation::findWatch } } } - Pstream::scatter(index); + Pstream::scatter(index); //, Pstream::msgType(), comm_); return index; } @@ -1764,11 +2652,11 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::getFile ) const { fileName fName; - if (Pstream::master()) + if (Pstream::master()) // comm_)) { fName = monitor().getFile(watchIndex); } - Pstream::scatter(fName); + Pstream::scatter(fName); //, Pstream::msgType(), comm_); return fName; } @@ -1779,7 +2667,7 @@ void Foam::fileOperations::masterUncollatedFileOperation::updateStates const bool syncPar ) const { - if (Pstream::master()) + if (Pstream::master()) // comm_)) { monitor().updateStates(true, false); } @@ -1793,11 +2681,11 @@ Foam::fileOperations::masterUncollatedFileOperation::getState ) const { unsigned int state = fileMonitor::UNMODIFIED; - if (Pstream::master()) + if (Pstream::master()) // comm_)) { state = monitor().getState(watchFd); } - Pstream::scatter(state); + Pstream::scatter(state); //, Pstream::msgType(), comm_); return fileMonitor::fileState(state); } @@ -1807,7 +2695,7 @@ void Foam::fileOperations::masterUncollatedFileOperation::setUnmodified const label watchFd ) const { - if (Pstream::master()) + if (Pstream::master()) // comm_)) { monitor().setUnmodified(watchFd); } diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H index c5478c8cea1e51816428018ffedf47e66c687ffe..77eba09a7c75a6d25a08e68d89a65a988d885fb7 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H @@ -28,6 +28,31 @@ Description fileOperations that performs all file operations on the master processor. Requires the calls to be parallel synchronised! + Limitations: - no /processor in filename + - no /uniform/ in the filename + + The main logic is in ::filePath which returns a + - same path on all processors. This can either be a global file + (system/controlDict, processorXXX/0/uniform/) or a collated file + (processors/0/p) + - same path on all processors of the local communicator + (processors4_0-1/0/p) + - different path on all processors (processor0/0/p) + + system/controlDict: + filePath worldmaster: <globalRoot>/system/controlDict + localmaster: ,, + slave : ,, + + processor0/uniform/time + filePath worldmaster: <globalRoot>/processorXXX/uniform/time + localmaster: ,, + slave : ,, + + processors0/0/p + processors10/0/p + processors10_2-4/0/p + \*---------------------------------------------------------------------------*/ #ifndef fileOperations_masterUncollatedFileOperation_H @@ -36,6 +61,9 @@ Description #include "fileOperation.H" #include "OSspecific.H" #include "HashPtrTable.H" +#include "Switch.H" +#include "unthreadedInitialise.H" +#include "boolList.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -59,6 +87,9 @@ protected: // Protected data + //- Any communicator allocated by me + const label myComm_; + //- Cached times for a given directory mutable HashPtrTable<instantList> times_; @@ -354,18 +385,29 @@ protected: // Private Member Functions + //- Get the list of processors that are part of this communicator + static labelList subRanks(const label n); + template<class Type> - Type scatterList(const UList<Type>&) const; + Type scatterList(const UList<Type>&, const int, const label comm) const; template<class Type, class fileOp> - Type masterOp(const fileName&, const fileOp& fop) const; + Type masterOp + ( + const fileName&, + const fileOp& fop, + const int tag, + const label comm + ) const; template<class Type, class fileOp> Type masterOp ( const fileName&, const fileName&, - const fileOp& fop + const fileOp& fop, + const int tag, + const label comm ) const; //- Equivalent of Time::findInstance @@ -375,26 +417,32 @@ protected: const instant& t ); - //- Search for object; return info on how it was found + //- Search (locally!) for object; return info on how it was found. + // Does not do any parallel communication. // checkGlobal : also check undecomposed case - // isFile : true:check for file; false:check for directory - fileName filePathInfo + // isFile : true:check for file false:check for directory + // searchType : how was found + // processorsDir : name of processor directory + // instance : instance + virtual fileName filePathInfo ( const bool checkGlobal, const bool isFile, - const IOobject& io, + const IOobject&, const bool search, - pathType&, - word& + pathType& searchType, + word& processorsDir, + word& instance ) const; //- Construct filePath - static fileName objectPath + fileName localObjectPath ( const IOobject&, - const pathType&, - const word& - ); + const pathType& searchType, + const word& processorsDir, + const word& instancePath + ) const; //- Read file contents and send to processors static void readAndSend @@ -405,6 +453,29 @@ protected: PstreamBuffers& pBufs ); + //- Detect file (possibly compressed), read file contents and send + // to processors + static void readAndSend + ( + const fileName& fName, + const labelUList& procs, + PstreamBuffers& pBufs + ); + + //- Read files on comms master + static autoPtr<ISstream> read + ( + IOobject& io, + const label comm, + const bool uniform, // on comms master only + const fileNameList& filePaths, // on comms master only + const boolList& procValid // on comms master only + ); + + //- Helper: check IO for local existence. Like filePathInfo but + // without parent searchign and instance searching + bool exists(const dirIndexList&, IOobject& io) const; + public: @@ -425,9 +496,12 @@ public: //- Construct null masterUncollatedFileOperation(const bool verbose); + //- Construct from communicator + masterUncollatedFileOperation(const label comm, const bool verbose); + //- Destructor - virtual ~masterUncollatedFileOperation() = default; + virtual ~masterUncollatedFileOperation(); // Member Functions @@ -682,30 +756,19 @@ public: //- Get sorted list of times virtual instantList findTimes(const fileName&, const word&) const; + //- Find instance where IOobject is. Fails if cannot be found + // and readOpt() is MUST_READ/MUST_READ_IF_MODIFIED. Otherwise + // returns stopInstance. + virtual IOobject findInstance + ( + const IOobject& io, + const scalar startValue, + const word& stopInstance + ) const; + //- Callback for time change virtual void setTime(const Time&) const; - //- root+casename with any 'processorXXX' replaced by 'processsors' - static fileName processorsCasePath(const IOobject&); - - //- Like io.path with provided instance and any 'processorXXX' - // replaced by 'processsors' - static fileName processorsPath(const IOobject&, const word&); - - //- Operating on fileName: replace processorXXX with processors - static fileName processorsPath(const fileName&); - - //- Split fileName into part before processor and part after. - // Returns -1 or processor number. Use with care. - // - path/"processor" + Foam::name(proci)/local reconstructs input - // - path/"processors"/local reconstructs processors equivalence - static label splitProcessorPath - ( - const fileName&, - fileName& path, - fileName& local - ); - //- Return cached times const HashPtrTable<instantList>& times() const { @@ -714,6 +777,28 @@ public: }; +/*---------------------------------------------------------------------------*\ + Class masterUncollatedFileOperationInitialise Declaration +\*---------------------------------------------------------------------------*/ + +class masterUncollatedFileOperationInitialise +: + public unthreadedInitialise +{ +public: + + // Constructors + + //- Construct from components + masterUncollatedFileOperationInitialise(int& argc, char**& argv); + + + //- Destructor + virtual ~masterUncollatedFileOperationInitialise() + {} +}; + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // } // End namespace fileOperations diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperationTemplates.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperationTemplates.C index 5c1a5b0716446688561569053356728cd65252dc..6d38bdd9193ccb3ef5640a550d30757dbb1daa17 100644 --- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperationTemplates.C +++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperationTemplates.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2017-2018 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -31,14 +31,16 @@ License template<class Type> Type Foam::fileOperations::masterUncollatedFileOperation::scatterList ( - const UList<Type>& masterLst + const UList<Type>& masterLst, + const int tag, + const label comm ) const { // TBD: more efficient scatter - PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking); - if (Pstream::master()) + PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking, tag, comm); + if (Pstream::master(comm)) { - for (label proci = 1; proci < Pstream::nProcs(); proci++) + for (label proci = 1; proci < Pstream::nProcs(comm); proci++) { UOPstream os(proci, pBufs); os << masterLst[proci]; @@ -48,9 +50,9 @@ Type Foam::fileOperations::masterUncollatedFileOperation::scatterList Type myResult; - if (Pstream::master()) + if (Pstream::master(comm)) { - myResult = masterLst[Pstream::myProcNo()]; + myResult = masterLst[Pstream::myProcNo(comm)]; } else { @@ -65,21 +67,25 @@ template<class Type, class fileOp> Type Foam::fileOperations::masterUncollatedFileOperation::masterOp ( const fileName& fName, - const fileOp& fop + const fileOp& fop, + const int tag, + const label comm ) const { if (IFstream::debug) { - Pout<< "masterUncollatedFileOperation : Operation on " << fName << endl; + Pout<< "masterUncollatedFileOperation::masterOp : Operation " + << typeid(fileOp).name() + << " on " << fName << endl; } if (Pstream::parRun()) { - List<fileName> filePaths(Pstream::nProcs()); - filePaths[Pstream::myProcNo()] = fName; - Pstream::gatherList(filePaths); + List<fileName> filePaths(Pstream::nProcs(comm)); + filePaths[Pstream::myProcNo(comm)] = fName; + Pstream::gatherList(filePaths, tag, comm); - List<Type> result(Pstream::nProcs()); - if (Pstream::master()) + List<Type> result(filePaths.size()); + if (Pstream::master(comm)) { result = fop(filePaths[0]); for (label i = 1; i < filePaths.size(); i++) @@ -91,7 +97,7 @@ Type Foam::fileOperations::masterUncollatedFileOperation::masterOp } } - return scatterList(result); + return scatterList(result, tag, comm); } else { @@ -105,7 +111,9 @@ Type Foam::fileOperations::masterUncollatedFileOperation::masterOp ( const fileName& src, const fileName& dest, - const fileOp& fop + const fileOp& fop, + const int tag, + const label comm ) const { if (IFstream::debug) @@ -115,16 +123,16 @@ Type Foam::fileOperations::masterUncollatedFileOperation::masterOp } if (Pstream::parRun()) { - List<fileName> srcs(Pstream::nProcs()); - srcs[Pstream::myProcNo()] = src; - Pstream::gatherList(srcs); + List<fileName> srcs(Pstream::nProcs(comm)); + srcs[Pstream::myProcNo(comm)] = src; + Pstream::gatherList(srcs, tag, comm); - List<fileName> dests(Pstream::nProcs()); - dests[Pstream::myProcNo()] = dest; - Pstream::gatherList(dests); + List<fileName> dests(srcs.size()); + dests[Pstream::myProcNo(comm)] = dest; + Pstream::gatherList(dests, tag, comm); - List<Type> result(Pstream::nProcs()); - if (Pstream::master()) + List<Type> result(Pstream::nProcs(comm)); + if (Pstream::master(comm)) { result = fop(srcs[0], dests[0]); for (label i = 1; i < srcs.size(); i++) @@ -136,7 +144,7 @@ Type Foam::fileOperations::masterUncollatedFileOperation::masterOp } } - return scatterList(result); + return scatterList(result, tag, comm); } else { diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C index f0c7d40a460b485049385a7ffcec68e57515e7b2..f96ea886a7a7a286326f2afa416b4c13d6083c01 100644 --- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C +++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C @@ -27,9 +27,9 @@ License #include "Time.H" #include "Fstream.H" #include "addToRunTimeSelectionTable.H" -#include "masterUncollatedFileOperation.H" #include "decomposedBlockData.H" #include "dummyISstream.H" +#include "unthreadedInitialise.H" /* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */ @@ -39,6 +39,15 @@ namespace fileOperations { defineTypeNameAndDebug(uncollatedFileOperation, 0); addToRunTimeSelectionTable(fileOperation, uncollatedFileOperation, word); + + // Mark as not needing threaded mpi + addNamedToRunTimeSelectionTable + ( + fileOperationInitialise, + unthreadedInitialise, + word, + uncollated + ); } } @@ -102,17 +111,20 @@ Foam::fileName Foam::fileOperations::uncollatedFileOperation::filePathInfo // Check if parallel "procesors" directory if (io.time().processorCase()) { - fileName path = fileOperations::masterUncollatedFileOperation:: - processorsPath + tmpNrc<dirIndexList> pDirs ( - io, - io.instance() + lookupProcessorsPath(io.objectPath()) ); - fileName objectPath = path/io.name(); - - if (isFileOrDir(isFile, objectPath)) + forAll(pDirs(), i) { - return objectPath; + const fileName& pDir = pDirs()[i].first(); + fileName objPath = + processorsPath(io, io.instance(), pDir) + /io.name(); + if (objPath != objectPath && isFileOrDir(isFile, objPath)) + { + return objPath; + } } } @@ -153,6 +165,8 @@ Foam::fileOperations::uncollatedFileOperation::uncollatedFileOperation ( const bool verbose ) +: + fileOperation(Pstream::worldComm) { if (verbose) { @@ -533,15 +547,7 @@ Foam::fileOperations::uncollatedFileOperation::readStream { // Analyse the objectpath to find out the processor we're trying // to access - fileName path; - fileName local; - label proci = fileOperations::masterUncollatedFileOperation:: - splitProcessorPath - ( - io.objectPath(), - path, - local - ); + label proci = detectProcessorPath(io.objectPath()); if (proci == -1) { @@ -551,6 +557,26 @@ Foam::fileOperations::uncollatedFileOperation::readStream << exit(FatalIOError); } + // Analyse the fileName for any processor subset. Note: this + // should really be part of filePath() which should return + // both file and index in file. + fileName path, procDir, local; + label groupStart, groupSize, nProcs; + splitProcessorPath + ( + fName, + path, + procDir, + local, + groupStart, + groupSize, + nProcs + ); + if (groupStart != -1 && groupSize > 0) + { + proci = proci-groupStart; + } + // Read data and return as stream return decomposedBlockData::readBlock(proci, isPtr(), io); } @@ -570,8 +596,8 @@ bool Foam::fileOperations::uncollatedFileOperation::read { if (debug) { - Pout<< "uncollatedFileOperation::read() : " - << "reading object " << io.objectPath() + Pout<< "uncollatedFileOperation::read :" + << " Reading object " << io.objectPath() << " from file " << endl; } diff --git a/src/OpenFOAM/include/OSspecific.H b/src/OpenFOAM/include/OSspecific.H index c5077f01a2dc8f2442f6279bdebe18f2120d569c..1fd54ec0d501f3ab7b51f2085ac47e22a8e0316b 100644 --- a/src/OpenFOAM/include/OSspecific.H +++ b/src/OpenFOAM/include/OSspecific.H @@ -256,33 +256,6 @@ bool dlSymFound(void* handle, const std::string& symbol); fileNameList dlLoaded(); -// Thread handling - -//- Allocate a thread -label allocateThread(); - -//- Start a thread -void createThread(const label, void *(*start_routine) (void *), void *arg); - -//- Wait for thread -void joinThread(const label); - -//- Delete a thread -void freeThread(const label); - -//- Allocate a mutex variable -label allocateMutex(); - -//- Lock a mutex variable -void lockMutex(const label); - -//- Unlock a mutex variable -void unlockMutex(const label); - -//- Free a mutex variable -void freeMutex(const label); - - // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // } // End namespace Foam diff --git a/src/OpenFOAM/memory/tmp/tmpNrc.H b/src/OpenFOAM/memory/tmp/tmpNrc.H index 30eabf397a86ed1742a5315a8946b53ca7211d71..319754ce77d48f520c3c9b4cb12d826221e4a370 100644 --- a/src/OpenFOAM/memory/tmp/tmpNrc.H +++ b/src/OpenFOAM/memory/tmp/tmpNrc.H @@ -67,7 +67,7 @@ class tmpNrc mutable T* ptr_; //- The type (managed pointer | const-reference object) - refType type_; + mutable refType type_; public: @@ -160,6 +160,9 @@ public: //- delete object and set pointer to nullptr inline void clear() const; + //- Swaps the managed object with other tmpNrc. + inline void swap(tmpNrc<T>& other) noexcept; + // Member operators diff --git a/src/Pstream/dummy/UPstream.C b/src/Pstream/dummy/UPstream.C index 797f3901791fcba5963b95da321bfeffe4a39cd7..c3ca30631bbcc107c2fad034f7323b8c346bf40f 100644 --- a/src/Pstream/dummy/UPstream.C +++ b/src/Pstream/dummy/UPstream.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation \\/ M anipulation | Copyright (C) 2016 OpenCFD Ltd. ------------------------------------------------------------------------------- License @@ -42,7 +42,7 @@ bool Foam::UPstream::initNull() } -bool Foam::UPstream::init(int& argc, char**& argv) +bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread) { FatalErrorInFunction << "The dummy Pstream library cannot be used in parallel mode" diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index de340c1c36e6082cb0c13f88f7b35f836a508e85..4e68dc8bf717259789bcd6e654c04da89687d4ec 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -101,7 +101,7 @@ bool Foam::UPstream::initNull() } -bool Foam::UPstream::init(int& argc, char**& argv) +bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread) { int flag = 0; @@ -129,17 +129,16 @@ bool Foam::UPstream::init(int& argc, char**& argv) //MPI_Init(&argc, &argv); - int wanted_thread_support = MPI_THREAD_SINGLE; - if (fileOperations::collatedFileOperation::maxThreadFileBufferSize > 0) - { - wanted_thread_support = MPI_THREAD_MULTIPLE; - } int provided_thread_support; MPI_Init_thread ( &argc, &argv, - wanted_thread_support, + ( + needsThread + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE + ), &provided_thread_support ); diff --git a/tutorials/IO/fileHandler/Allclean b/tutorials/IO/fileHandler/Allclean index be7ddb068847225ad69704840909a7b0d05a5466..d7d5f0c37034a2ddd919fc826239dcaa44032753 100755 --- a/tutorials/IO/fileHandler/Allclean +++ b/tutorials/IO/fileHandler/Allclean @@ -1,7 +1,11 @@ #!/bin/sh -cd ${0%/*} || exit 1 # Run from this directory -. $WM_PROJECT_DIR/bin/tools/CleanFunctions # Tutorial clean functions +cd ${0%/*} || exit 1 # Run from this directory + +# Source tutorial run functions +. $WM_PROJECT_DIR/bin/tools/CleanFunctions cleanCase +rm -rf machineA +rm -rf machineB #------------------------------------------------------------------------------ diff --git a/tutorials/IO/fileHandler/Allrun b/tutorials/IO/fileHandler/Allrun index 52d50e93fafbb3dc5f51643d6b7d6e54fda19909..ed0952fe54b252e1b754fefc8199ec48d6b73b17 100755 --- a/tutorials/IO/fileHandler/Allrun +++ b/tutorials/IO/fileHandler/Allrun @@ -22,13 +22,33 @@ runApplication decomposePar -fileHandler collated runParallel $application -fileHandler collated runApplication reconstructPar -latestTime -fileHandler collated -#- Delete collated files -rm -rf processors - #- Test writing uncollated format -runApplication -s uncollated decomposePar -fileHandler uncollated +runApplication -s uncollated decomposePar -fileHandler uncollated -force runParallel -s uncollated $application -fileHandler uncollated + +#- Test uncollated+distributed running: copy to different roots +rm -rf machineA/fileHandler +mkdir -p machineA/fileHandler +( cd machineA/fileHandler && \ + cp -r ../../processor[0-1] . && \ + cp -r ../../system . && \ + mkdir -p constant && cp ../../constant/* constant +) +#- Note: slave node does not need constant&system since these are global +rm -rf machineB/fileHandler +mkdir -p machineB/fileHandler +( cd machineB/fileHandler && \ + cp -r ../../processor[2-3] . +) +#- Run with different roots +( d=$PWD && \ + cd machineA/fileHandler && \ + runParallel -s multiRoot $application \ + -fileHandler masterUncollated -ioRanks '(0 2)' \ + -roots "(\"$d/machineA\" \"$d/machineB\" \"$d/machineB\")" +) + #- Restart from uncollated runParallel -s collated $application -fileHandler collated runApplication -s collated reconstructPar -latestTime -fileHandler collated @@ -36,4 +56,50 @@ runApplication -s collated reconstructPar -latestTime -fileHandler collated #- Convert the parallel format to uncollated runParallel foamFormatConvert -fileHandler uncollated +#- Restart with multiple IO ranks +runParallel -s multiCollated \ + $application -fileHandler collated -ioRanks '(0 2)' + +#- Reconstruct the multi-rank format. Delete the collated directory +# since conflicts with the multi-collated directory +rm -rf processors4 +runApplication -s multiCollated reconstructPar -latestTime \ + -fileHandler collated -ioRanks '(0 2)' + +#- Convert the multi-rank format to uncollated +runParallel -s uncollated foamFormatConvert -fileHandler uncollated + +#- Restart from multiCollated using collated +runParallel -s uncollated_from_multiCollated \ + $application -fileHandler uncollated + + + +#- Test collated+distributed running: copy to different roots +# Important: make sure to copy uniform since we're copying it + +#- Delete all processor directories +runApplication -s collated decomposePar \ + -fileHandler collated -force -copyUniform + +rm -rf machineA/fileHandler +mkdir -p machineA/fileHandler +( cd machineA/fileHandler && \ + cp -r ../../processor* . && \ + cp -r ../../system . && \ + mkdir -p constant && cp ../../constant/* constant +) +#- Note: slave node does not need constant&system since these are global +rm -rf machineB/fileHandler +mkdir -p machineB/fileHandler + +#- Run with different roots +( d=$PWD && \ + cd machineA/fileHandler && \ + runParallel -s distributed_multiCollated $application \ + -fileHandler collated -ioRanks '(0 2)' \ + -roots "(\"$d/machineA\" \"$d/machineB\" \"$d/machineB\")" +) + + #------------------------------------------------------------------------------ diff --git a/tutorials/IO/fileHandler/system/controlDict b/tutorials/IO/fileHandler/system/controlDict index d2090358217d0e658dc748b212bb992b102d9cc6..135f64524bd24380457f9da5686fb50d2a51d5fd 100644 --- a/tutorials/IO/fileHandler/system/controlDict +++ b/tutorials/IO/fileHandler/system/controlDict @@ -37,7 +37,7 @@ writeFormat binary; writePrecision 6; -writeCompression uncompressed; +writeCompression false; timeFormat general; @@ -45,9 +45,4 @@ timePrecision 6; runTimeModifiable yes; -// OptimisationSwitches -// { -// fileHandler collated; -// } - // ************************************************************************* //