diff --git a/applications/test/memoryPool1/Make/files b/applications/test/memoryPool1/Make/files new file mode 100644 index 0000000000000000000000000000000000000000..200520b0ad760f1bbaa0a7ae4529fb7e1601530f --- /dev/null +++ b/applications/test/memoryPool1/Make/files @@ -0,0 +1,3 @@ +Test-memoryPool1.cxx + +EXE = $(FOAM_USER_APPBIN)/Test-memoryPool1 diff --git a/applications/test/memoryPool1/Make/options b/applications/test/memoryPool1/Make/options new file mode 100644 index 0000000000000000000000000000000000000000..18e6fe47afacb902cddccf82632772447704fd88 --- /dev/null +++ b/applications/test/memoryPool1/Make/options @@ -0,0 +1,2 @@ +/* EXE_INC = */ +/* EXE_LIBS = */ diff --git a/applications/test/memoryPool1/Test-memoryPool1.cxx b/applications/test/memoryPool1/Test-memoryPool1.cxx new file mode 100644 index 0000000000000000000000000000000000000000..bcb9bc0834ff2a57ec5f9348a5c29b13c49a598a --- /dev/null +++ b/applications/test/memoryPool1/Test-memoryPool1.cxx @@ -0,0 +1,210 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2025 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +\*---------------------------------------------------------------------------*/ + +#include "argList.H" +#include "MemoryPool.H" +#include "IOstreams.H" + +// options +int min_size = 5; +bool use_aligned_alloc(true); +bool use_aligned_dealloc(true); +bool use_const_alignment(false); + +// Some type of selector +template<class IntType> +inline bool use_pool(IntType n) noexcept +{ + return (n >= IntType(min_size)); +} + + +//- Default alignment that is size-independent +inline std::align_val_t constexpr my_align() noexcept +{ + return std::align_val_t(64); +} + + +//- Default alignment that is size-dependent (depends on number of elements) +template<class IntType> +inline std::align_val_t constexpr my_align(IntType n) noexcept +{ + return std::align_val_t(n > 32 ? 256 : 16); +} + + +//- Allocate from memory pool (if active) or aligned/normal +template<class T, class IntType> +inline T* my_allocate(IntType n) +{ + std::cerr<< "my_allocate(" << n << ")\n"; + + if + ( + void *pool_ptr + ( + // Consider memory pool for large amounts of data + use_pool(n) + ? Foam::MemoryPool::try_allocate(sizeof(T)*n) + : nullptr + ); + pool_ptr + ) + { + // Placement new + return new (pool_ptr) T[n]; + } + else if (use_aligned_alloc) + { + if (use_const_alignment) + { + // Alignment is size-independent + return new (my_align()) T[n]; + } + else + { + // Alignment is size-dependent + return new (my_align(n)) T[n]; + } + } + else + { + // Plain new + return new T[n]; + } +} + + +//- Deallocate from memory pool or aligned/normal +template<class T, class IntType> +inline void my_deallocate(T* ptr, IntType n) +{ + std::cerr<< "my_deallocate(" << n << ")\n"; + + if (ptr) + { + if (!Foam::MemoryPool::try_deallocate(ptr)) + { + if (use_aligned_dealloc) + { + if (use_const_alignment) + { + // Alignment is size-independent + ::operator delete[](ptr, my_align()); + } + else + { + // Alignment is size-dependent + ::operator delete[](ptr, my_align(n)); + } + } + else + { + // Plain new + delete[] ptr; + } + } + } +} + + +using namespace Foam; + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Main program: + +int main(int argc, char *argv[]) +{ + argList::noCheckProcessorDirectories(); + argList::addOption + ( + "min-size", + "INT", + "Min size for memory pool (default: 5)" + ); + argList::addOption + ( + "count", + "INT", + "Number of elements to test (default: 10)" + ); + argList::addBoolOption + ( + "no-align-alloc", + "Do not use aligned alloc (default: false)" + ); + argList::addBoolOption + ( + "no-align-dealloc", + "Do not use aligned dealloc (default: false)" + ); + argList::addBoolOption + ( + "const-align", + "Use size-independent alignment (default: false)" + ); + + #include "setRootCase.H" + + label count(10); + + args.readIfPresent("count", count); + args.readIfPresent("min-size", min_size); + + use_aligned_alloc = !args.found("no-align-alloc"); + use_aligned_dealloc = !args.found("no-align-dealloc"); + use_const_alignment = args.found("const-align"); + + + Info<< "constant align: " << int(my_align()) << nl + << "variable align: " << int(my_align(count)) + << " for " << count << " elements" << nl + << nl; + + { + using T = double; + label len = count; + + UList<T> list(my_allocate<T>(len), len); + + Info<< "List ptr: " << Foam::name(list.data()) << nl; + + list = 100; + + Info<< "List: " << list << nl; + + my_deallocate(list.data(), len); + + list = UList<T>(); + } + + return 0; +} + + +// ************************************************************************* // diff --git a/etc/bashrc b/etc/bashrc index 60030f8ce9cd5ced9f291672e201448c4c8492a1..9fc456394416b141804d7e5c3da579b3a05875e5 100644 --- a/etc/bashrc +++ b/etc/bashrc @@ -132,6 +132,11 @@ projectDir="$HOME/OpenFOAM/OpenFOAM-$WM_PROJECT_VERSION" # projectDir="@PROJECT_DIR@" : # Safety statement (if the user removed all fallback values) +# [FOAM_MEMORY_POOL] - Optional memory management +# - overrides the 'memory_pool' etc/controlDict entry +# = "true | false | host [size=nn] [incr=nn]" +#export FOAM_MEMORY_POOL="host" + # [FOAM_SIGFPE] - Trap floating-point exceptions. # - overrides the 'trapFpe' controlDict entry # = true | false diff --git a/etc/controlDict b/etc/controlDict index 74d85e59017bac8b49d082aaf6dcfcf5551bf012..884a9b9e96fedbed54e5db715bd62df4cada5b46 100644 --- a/etc/controlDict +++ b/etc/controlDict @@ -1,7 +1,7 @@ /*--------------------------------*- C++ -*----------------------------------*\ | ========= | | | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | -| \\ / O peration | Version: v2412 | +| \\ / O peration | Version: v2506 | | \\ / A nd | Website: www.openfoam.com | | \\/ M anipulation | | \*---------------------------------------------------------------------------*/ @@ -213,6 +213,9 @@ OptimisationSwitches // Other // ===== + // Optional memory management (sizing in MB) + // memory_pool "host; size=1024; incr=5" + // Trap floating point exception. // Can override with FOAM_SIGFPE env variable (true|false) trapFpe 1; diff --git a/etc/cshrc b/etc/cshrc index b36054eabd4975b995b4f0d52b547b2fb6fab26e..5e19865b00ba584ed5b4f6c6ff5fca73f18622ec 100644 --- a/etc/cshrc +++ b/etc/cshrc @@ -134,6 +134,11 @@ set projectDir=`lsof +p $$ |& sed -ne 's#^[^/]*##;\@/'"$projectName"'[^/]*/etc/c # Or optionally hard-coded (eg, with autoconfig) # set projectDir="@PROJECT_DIR@" +# [FOAM_MEMORY_POOL] - Optional memory management +# - overrides the 'memory_pool' etc/controlDict entry +# = "true | false | host [size=nn] [incr=nn]" +#setenv FOAM_MEMORY_POOL "host" + # [FOAM_SIGFPE] - Trap floating-point exceptions. # - overrides the 'trapFpe' controlDict entry # = true | false diff --git a/src/OSspecific/MSwindows/Make/files b/src/OSspecific/MSwindows/Make/files index 7c0823f3fcee6c7bada9f3bd48ebd1e686594e07..50c0c9bf652de1a5d2eb1ed70d11a49348e60f5f 100644 --- a/src/OSspecific/MSwindows/Make/files +++ b/src/OSspecific/MSwindows/Make/files @@ -3,6 +3,8 @@ MSwindows.C cpuInfo/cpuInfo.C memInfo/memInfo.C +memory/MemoryPool.cxx + signals/sigFpe.cxx signals/sigInt.cxx signals/sigQuit.cxx diff --git a/src/OSspecific/MSwindows/memory/MemoryPool.cxx b/src/OSspecific/MSwindows/memory/MemoryPool.cxx new file mode 100644 index 0000000000000000000000000000000000000000..5bf42da1ff6b3bfefda2bbe2cff43f381b0c8345 --- /dev/null +++ b/src/OSspecific/MSwindows/memory/MemoryPool.cxx @@ -0,0 +1,82 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2025 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +\*---------------------------------------------------------------------------*/ + +#include "MemoryPool.H" + +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // + +// bool Foam::MemoryPool::create(const std::string& ctrl, bool verbose) +// { +// return false; +// } + + +bool Foam::MemoryPool::create(bool verbose) +{ + return false; +} + + +void Foam::MemoryPool::destroy(bool verbose) +{} + + +bool Foam::MemoryPool::active() noexcept +{ + return false; +} + + +bool Foam::MemoryPool::suspend() noexcept +{ + return false; +} + + +void Foam::MemoryPool::resume() noexcept +{} + + +bool Foam::MemoryPool::is_pool(void* ptr) +{ + return false; +} + + +void* Foam::MemoryPool::try_allocate(std::size_t nbytes) +{ + return nullptr; +} + + +bool Foam::MemoryPool::try_deallocate(void* ptr) +{ + return (!ptr); +} + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // diff --git a/src/OSspecific/POSIX/Allwmake b/src/OSspecific/POSIX/Allwmake index 7bc223e001502f1431b0e73f4732b40d9d033da1..c02a48f9cfc3a6a44b889768cd4d49cf0b760fc8 100755 --- a/src/OSspecific/POSIX/Allwmake +++ b/src/OSspecific/POSIX/Allwmake @@ -2,6 +2,7 @@ cd "${0%/*}" || exit # Run from this directory targetType=libo # Preferred library type . ${WM_PROJECT_DIR:?}/wmake/scripts/AllwmakeParseArguments $* +. ${WM_PROJECT_DIR:?}/wmake/scripts/have_umpire #------------------------------------------------------------------------------ # Hack for MacOS (with Gcc). @@ -59,6 +60,47 @@ then export COMP_FLAGS="-DFOAM_USE_INOTIFY" fi +#------------------------------------------------------------------------------ + +# Have -lumpire, but also -lcamp etc. +# Also need to follow the link order +get_umpire_libs() +{ + if [ -d "${UMPIRE_LIB_DIR}" ] + then + set -- $( + # Expected link order + for name in umpire fmt camp + do + [ -f "$UMPIRE_LIB_DIR/lib${name}.a" ] && echo "-l$name" + done + ) + echo "$@" + else + echo + fi +} + + +if have_umpire +then + libNames="$(get_umpire_libs)" + + if [ -n "$libNames" ] + then + echo " found umpire -- enabling memory pool interface" 1>&2 + echo " umpire libs: $libNames" 1>&2 + + COMP_FLAGS="$COMP_FLAGS -DFOAM_USE_UMPIRE -I${UMPIRE_INC_DIR}" + LINK_FLAGS="$LINK_FLAGS -L${UMPIRE_LIB_DIR} $libNames" + export COMP_FLAGS LINK_FLAGS + else + echo " expecting umpire, but did not resolve the libraries" 1>&2 + fi +fi + +#------------------------------------------------------------------------------ + # Make object (non-shared by default) # Never want/need openmp, especially for static objects wmake -no-openmp $targetType diff --git a/src/OSspecific/POSIX/Make/files b/src/OSspecific/POSIX/Make/files index 29c8588445158e0820febf024a803f407fe03460..ed5d69dc73b037b6bdcce707961da2d7d6e80aaf 100644 --- a/src/OSspecific/POSIX/Make/files +++ b/src/OSspecific/POSIX/Make/files @@ -4,6 +4,8 @@ cpuInfo/cpuInfo.C cpuTime/cpuTimePosix.C memInfo/memInfo.C +memory/MemoryPool.cxx + signals/sigFpe.cxx signals/sigSegv.cxx signals/sigInt.cxx diff --git a/src/OSspecific/POSIX/Make/options b/src/OSspecific/POSIX/Make/options index 3f86d412a62b7c58d486f100cbde05dcc9de5827..2e18d513bdb61a9a7bf827570ef63deefb05e5d1 100644 --- a/src/OSspecific/POSIX/Make/options +++ b/src/OSspecific/POSIX/Make/options @@ -1 +1,4 @@ -EXE_INC = $(COMP_FLAGS) +/* umpire uses old-style cast etc */ +EXE_INC = $(COMP_FLAGS) $(c++LESSWARN) + +LIBO_LIBS = $(LINK_FLAGS) diff --git a/src/OSspecific/POSIX/memory/MemoryPool.cxx b/src/OSspecific/POSIX/memory/MemoryPool.cxx new file mode 100644 index 0000000000000000000000000000000000000000..5a799aee0c69807df889fa4968ba4cbb34d3fc1f --- /dev/null +++ b/src/OSspecific/POSIX/memory/MemoryPool.cxx @@ -0,0 +1,502 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2025 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +\*---------------------------------------------------------------------------*/ + +#include "MemoryPool.H" +#include "debug.H" +#include "dictionary.H" +#include "sigFpe.H" +#include "OSspecific.H" // For getEnv + +// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // + +#ifdef FOAM_USE_UMPIRE + +// #include <cerrno> +#include <cinttypes> +#include <tuple> + +#include "umpire/Allocator.hpp" +#include "umpire/ResourceManager.hpp" +#include "umpire/strategy/AlignedAllocator.hpp" +#include "umpire/strategy/DynamicPoolList.hpp" + +static bool disabled_(false); +static umpire::Allocator aligned_allocator; +static umpire::Allocator pooled_allocator; +static umpire::ResourceManager* manager_(nullptr); +static umpire::ResourceManager* suspended_(nullptr); + +#endif + +// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * // + +#ifdef FOAM_USE_UMPIRE + +namespace +{ + +// Different supported allocation types +enum class Types { undefined, none, host, device, managed }; + +typedef std::tuple<Types, std::size_t, std::size_t> ctrlTuple; + +// Extract key=INT, the key includes the '=' +int getIntParameter(const std::string& key, const std::string& ctrl) +{ + int val(0); + + const auto pos = ctrl.find(key); + + if (pos == std::string::npos) + { + return val; + } + + const char* buf = (ctrl.data() + pos + key.size()); + + char *endptr = nullptr; + errno = 0; + auto parsed = std::strtoimax(buf, &endptr, 10); + + if (errno || endptr == buf) + { + // Some type of error OR no conversion + } + else + { + val = int(parsed); + } + + return val; +} + + +ctrlTuple getControlValues(const std::string& ctrl) +{ + ctrlTuple result(Types::undefined, 0, 0); + + bool checkParam = false; + + // Also find things that look like Switch constants. + // Unfortunately need to do this manually since Switch::find() + // itself would not manage to parse something like "true; size=10" + + if (ctrl.empty()) + { + // Nothing => undefined + } + else if + ( + std::string::npos != ctrl.find("false") // ctrl.contains("false") + || std::string::npos != ctrl.find("off") // ctrl.contains("off") + || std::string::npos != ctrl.find("no") // ctrl.contains("no") + || std::string::npos != ctrl.find("none") // ctrl.contains("none") + ) + { + std::get<0>(result) = Types::none; + } + else if + ( + std::string::npos != ctrl.find("true") // ctrl.contains("true") + || std::string::npos != ctrl.find("on") // ctrl.contains("on") + || std::string::npos != ctrl.find("yes") // ctrl.contains("yes") + + || std::string::npos != ctrl.find("host") // ctrl.contains("host") + || std::string::npos != ctrl.find("system") // ctrl.contains("system") + ) + { + std::get<0>(result) = Types::host; + checkParam = true; + } + + // These need more testing + else if + ( + std::string::npos != ctrl.find("device") // ctrl.contains("device") + ) + { + std::get<0>(result) = Types::device; + checkParam = true; + } + else if + ( + std::string::npos != ctrl.find("managed") // ctrl.contains("managed") + ) + { + std::get<0>(result) = Types::managed; + checkParam = true; + } + + if (checkParam) + { + std::get<1>(result) = getIntParameter("size=", ctrl); + std::get<2>(result) = getIntParameter("incr=", ctrl); + } + + return result; +} + + +bool create_from(const ctrlTuple& controls, bool verbose) +{ + using namespace Foam; + + if (manager_ || suspended_) + { + // Already created + return true; + } + + // Type, initial size, increment + auto [which, size, incr] = controls; + + // std::cerr + // << "which=" << int(which) + // << ", size=" << int(size) + // << ", incr=" << int(incr) << '\n'; + + + constexpr size_t MegaByte(1024*1024); + + switch (which) + { + case Types::none : + { + if (verbose) + { + Info<< "memory pool : disabled" << nl; + } + break; + } + + case Types::host : + { + // Default sizing parameters + if (!size) size = 1024; + if (!incr) incr = 5; + + auto& rm = umpire::ResourceManager::getInstance(); + manager_ = &rm; + + aligned_allocator = + rm.makeAllocator<umpire::strategy::AlignedAllocator> + ( + "aligned_allocator", + rm.getAllocator("HOST"), + + // alignment + 256 + ); + + pooled_allocator = + rm.makeAllocator<umpire::strategy::DynamicPoolList> + ( + "openfoam_HOST_pool", + aligned_allocator, + + // initial block allocation size + (size*MegaByte), + + // incremental block allocation size + (incr*MegaByte) + ); + + if (verbose) + { + Info<< "memory pool : host (size=" + << int(size) << "MB, incr=" + << int(incr) << "MB)\n"; + } + break; + } + + case Types::device : + { + auto& rm = umpire::ResourceManager::getInstance(); + manager_ = &rm; + + aligned_allocator = rm.getAllocator("DEVICE"); + + pooled_allocator = + rm.makeAllocator<umpire::strategy::DynamicPoolList> + ( + "openfoam_DEVICE_pool", + aligned_allocator + ); + + if (verbose) + { + Info<< "memory pool : device" << nl; + } + break; + } + + case Types::managed : + { + // Default sizing parameters + if (!size) size = 10*1024; + if (!incr) incr = 10; + + auto& rm = umpire::ResourceManager::getInstance(); + manager_ = &rm; + + aligned_allocator = rm.getAllocator("UM"); + + pooled_allocator = + rm.makeAllocator<umpire::strategy::DynamicPoolList> + ( + "openfoam_UM_pool", + aligned_allocator, + + // initial block allocation size + (size*MegaByte), + + // incremental block allocation size + (incr*MegaByte) + ); + + if (verbose) + { + Info<< "memory pool : managed (size=" + << int(size) << "MB, incr=" + << int(incr) << "MB)\n"; + } + break; + } + + case Types::undefined : + { + break; + } + } + + return (which != Types::undefined && which != Types::none); +} + + +} // End anonymous namespace + +#endif // FOAM_USE_UMPIRE + + +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // + +// bool Foam::MemoryPool::create(const std::string& ctrl, bool verbose) +// { +// #ifdef FOAM_USE_UMPIRE +// if (manager_ || suspended_) +// { +// // Already created +// return true; +// } +// +// auto controls = getControlValues(ctrl); +// +// return create_from(controls, verbose); +// #else +// return false; +// #endif +// } + + +bool Foam::MemoryPool::create(bool verbose) +{ + #ifdef FOAM_USE_UMPIRE + if (disabled_) + { + // Disallowed + return false; + } + else if (manager_ || suspended_) + { + // Already created + return true; + } + + // First check environment + auto controls = getControlValues(Foam::getEnv("FOAM_MEMORY_POOL")); + + if (std::get<0>(controls) == Types::none) + { + // Disabled from environment - has highest priority + disabled_ = true; + } + + // Currently no easy way to handle <system>/controlDict... + + // Fallback from etc/controlDict + if (std::get<0>(controls) == Types::undefined) + { + // From central etc/controlDict + const auto& dict = Foam::debug::optimisationSwitches(); + + if (auto* eptr = dict.findStream("memory_pool", keyType::LITERAL)) + { + const token& firstToken = eptr->front(); + + if (firstToken.isStringType()) + { + controls = getControlValues(firstToken.stringToken()); + } + } + } + + return create_from(controls, verbose); + #else + return false; + #endif +} + + +void Foam::MemoryPool::destroy(bool verbose) +{ + // Nothing currently needed but could add in something like this: + + // if (manager_ || suspended_) + // { + // pooled_allocator.release(); + // } + + // However, need to find the proper sequence within + // Foam::exit() or UPstream::exit() ... +} + + +bool Foam::MemoryPool::active() noexcept +{ + #ifdef FOAM_USE_UMPIRE + return bool(manager_); + #else + return false; + #endif +} + + +bool Foam::MemoryPool::suspend() noexcept +{ + #ifdef FOAM_USE_UMPIRE + bool status(suspended_); + if (manager_) // <- and (!suspended_) + { + std::swap(manager_, suspended_); + } + return status; + #else + return false; + #endif +} + + +void Foam::MemoryPool::resume() noexcept +{ + #ifdef FOAM_USE_UMPIRE + if (suspended_) // <- and (!manager_) + { + std::swap(manager_, suspended_); + } + #endif +} + + +bool Foam::MemoryPool::is_pool(void* ptr) +{ + #ifdef FOAM_USE_UMPIRE + if (ptr) + { + if (manager_) + { + return manager_->hasAllocator(ptr); + } + else if (suspended_) + { + return suspended_->hasAllocator(ptr); + } + } + #endif + + return false; +} + + +void* Foam::MemoryPool::try_allocate(std::size_t nbytes) +{ + void* ptr = nullptr; + + #ifdef FOAM_USE_UMPIRE + if (manager_) + { + ptr = pooled_allocator.allocate(nbytes); + + // std::cerr<< "allocate(" << int(nbytes) << ")\n"; + + // Optionally fill with NaN (depends on current flags) + Foam::sigFpe::fillNan_if(ptr, nbytes); + + if (!ptr) + { + // Pout<< "umpire failed to allocate memory\n"; + } + } + #endif + + return ptr; +} + + +bool Foam::MemoryPool::try_deallocate(void* ptr) +{ + #ifdef FOAM_USE_UMPIRE + if (ptr) + { + if (manager_) + { + if (manager_->hasAllocator(ptr)) // <- ie, is_pool() + { + // std::cerr<< "deallocate()\n"; + manager_->deallocate(ptr); + return true; + } + } + else if (suspended_) + { + // Deallocate even if nominally suspended + + if (suspended_->hasAllocator(ptr)) // <- ie, is_pool() + { + // std::cerr<< "deallocate()\n"; + suspended_->deallocate(ptr); + return true; + } + } + } + #endif + + return (!ptr); +} + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // diff --git a/src/OpenFOAM/containers/Lists/List/List.C b/src/OpenFOAM/containers/Lists/List/List.C index c9707ef29bcc17563e3a84a284949ab82c664647..d7a45776d4a2a3206d9829081f2e4afa13e9fbb4 100644 --- a/src/OpenFOAM/containers/Lists/List/List.C +++ b/src/OpenFOAM/containers/Lists/List/List.C @@ -42,7 +42,7 @@ void Foam::List<T>::resize_copy(const label count, const label len) if (FOAM_LIKELY(len > 0)) { // With sign-check to avoid spurious -Walloc-size-larger-than - // const label oldLen = this->size_; + const label oldLen = this->size_; const label overlap = Foam::min(count, len); // Extra safety, not currently necessary: // const label overlap = Foam::min(Foam::min(count, oldLen), len); @@ -54,22 +54,22 @@ void Foam::List<T>::resize_copy(const label count, const label len) // Recover overlapping content when resizing this->size_ = len; - this->v_ = new T[len]; + this->v_ = ListPolicy::allocate<T>(len); // Can dispatch with // - std::execution::par_unseq // - std::execution::unseq std::move(old, (old + overlap), this->v_); - delete[] old; + ListPolicy::deallocate(old, oldLen); } else { // No overlapping content - delete[] old; + ListPolicy::deallocate(old, oldLen); this->size_ = len; - this->v_ = new T[len]; + this->v_ = ListPolicy::allocate<T>(len); } } else @@ -152,7 +152,7 @@ Foam::List<T>::List(const label len, Foam::zero) template<class T> Foam::List<T>::List(Foam::one, const T& val) : - UList<T>(new T[1], 1) + UList<T>(ListPolicy::allocate<T>(1), 1) { this->v_[0] = val; } @@ -161,7 +161,7 @@ Foam::List<T>::List(Foam::one, const T& val) template<class T> Foam::List<T>::List(Foam::one, T&& val) : - UList<T>(new T[1], 1) + UList<T>(ListPolicy::allocate<T>(1), 1) { this->v_[0] = std::move(val); } @@ -170,9 +170,9 @@ Foam::List<T>::List(Foam::one, T&& val) template<class T> Foam::List<T>::List(Foam::one, Foam::zero) : - UList<T>(new T[1], 1) + UList<T>(ListPolicy::allocate<T>(1), 1) { - this->v_[0] = Zero; + this->v_[0] = Foam::zero{}; } @@ -311,7 +311,8 @@ Foam::List<T>::List(DynamicList<T, SizeMin>&& list) template<class T> Foam::List<T>::~List() { - delete[] this->v_; + //TODO? May need to verify that size is accurate (for correct alignment) + ListPolicy::deallocate(this->v_, this->size_); } diff --git a/src/OpenFOAM/containers/Lists/List/ListI.H b/src/OpenFOAM/containers/Lists/List/ListI.H index 228df08392b4184c30a4ee614e7c7888e99db7d9..b8d292950b8a8889192d0753d85a9cf7cdb1a378 100644 --- a/src/OpenFOAM/containers/Lists/List/ListI.H +++ b/src/OpenFOAM/containers/Lists/List/ListI.H @@ -34,7 +34,7 @@ inline void Foam::List<T>::doAlloc() if (this->size_ > 0) { // With sign-check to avoid spurious -Walloc-size-larger-than - this->v_ = new T[this->size_]; + this->v_ = ListPolicy::allocate<T>(this->size_); } } @@ -136,11 +136,8 @@ inline Foam::autoPtr<Foam::List<T>> Foam::List<T>::clone() const template<class T> inline void Foam::List<T>::clear() { - if (this->v_) - { - delete[] this->v_; - this->v_ = nullptr; - } + ListPolicy::deallocate(this->v_, this->size_); + this->v_ = nullptr; this->size_ = 0; } diff --git a/src/OpenFOAM/containers/Lists/policy/ListPolicy.H b/src/OpenFOAM/containers/Lists/policy/ListPolicy.H index ed81c10d8f644142ab518b2e0baa2ba1ac10041d..513584f57e2d05fa714fdbd04b023ca9658169c5 100644 --- a/src/OpenFOAM/containers/Lists/policy/ListPolicy.H +++ b/src/OpenFOAM/containers/Lists/policy/ListPolicy.H @@ -34,7 +34,8 @@ Description #ifndef Foam_ListPolicy_H #define Foam_ListPolicy_H -#include <type_traits> +#include "MemoryPool.H" // Also includes <cstdint> +#include "contiguous.H" // Also includes <type_traits> // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -88,10 +89,168 @@ template<> struct no_linebreak<wordRe> : std::true_type {}; // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +//- Use size-independent alignment? +inline constexpr bool use_constant_alignment() noexcept +{ + // Define as hardcoded compile-time change: + return true; + // return false; +} + + +//- Should use aligned allocation for the given type? +// Benefits for field data (floating-point, ints, vectorspace), +// but avoid for char data, strings, pointers etc +template<class T> +inline constexpr bool is_aligned_type() noexcept +{ + return + ( + !std::is_enum_v<T> + && !std::is_pointer_v<T> + && !std::is_union_v<T> + && (sizeof(T) >= 4) // skip small data (eg, char) + && is_contiguous_v<T> + ); +} + + +//- True if size exceeds the (hardcoded) min-size for using the memory pool +template<class IntType> +inline constexpr bool use_memory_pool(IntType n) noexcept +{ + return (n >= IntType(3000)); +} + + +//- True if size exceeds the (hardcoded) min-size for offloading +template<class IntType> +inline constexpr bool use_offload(IntType n) noexcept +{ + return (n >= IntType(1000)); +} + + +// Note: can also specify __STDCPP_DEFAULT_NEW_ALIGNMENT__ +// but that is not more convenient... + +//- Default alignment that is size-independent +// Double the cache-line size is reportedly a good compromise (Leopold: AMD) +inline constexpr std::align_val_t default_alignment() noexcept +{ + return std::align_val_t(128); +} + + +//- Default alignment that is size-dependent (depends on number of elements) +template<class IntType> +inline constexpr std::align_val_t default_alignment(IntType n) noexcept +{ + return std::align_val_t(n > 200 ? 256 : 16); +} + + +//- Allocate from memory pool (if active), or aligned, or normal +template<class T, class IntType> +inline T* allocate(IntType n) +{ + if constexpr (ListPolicy::is_aligned_type<T>()) + { + if + ( + void *pool_ptr + ( + // Consider memory pool for large amounts of data + ListPolicy::use_memory_pool(n) + ? Foam::MemoryPool::try_allocate(sizeof(T)*n) + : nullptr + ); + pool_ptr + ) + { + // Placement new + return new (pool_ptr) T[n]; + } + else + { + if constexpr (ListPolicy::use_constant_alignment()) + { + // Alignment is size-independent + return new (ListPolicy::default_alignment()) T[n]; + } + else + { + // Alignment is size-dependent + return new (ListPolicy::default_alignment(n)) T[n]; + } + } + } + else + { + // Plain new + return new T[n]; + } +} + + +//- Deallocate from memory pool, or aligned, or normal +template<class T, class IntType> +inline void deallocate(T* ptr) +{ + if (ptr) + { + if constexpr (ListPolicy::is_aligned_type<T>()) + { + if (!Foam::MemoryPool::try_deallocate(ptr)) + { + // Alignment is size-independent (no size available) + ::operator delete[](ptr, ListPolicy::default_alignment()); + } + } + else + { + // Plain new + delete[] ptr; + } + } +} + + +//- Deallocate from memory pool, or aligned, or normal +template<class T, class IntType> +inline void deallocate(T* ptr, [[maybe_unused]] IntType n) +{ + if (ptr) + { + if constexpr (ListPolicy::is_aligned_type<T>()) + { + if (!Foam::MemoryPool::try_deallocate(ptr)) + { + if constexpr (ListPolicy::use_constant_alignment()) + { + // Alignment is size-independent + ::operator delete[](ptr, ListPolicy::default_alignment()); + } + else + { + // Alignment is size-dependent + ::operator delete[](ptr, ListPolicy::default_alignment(n)); + } + } + } + else + { + // Plain new + delete[] ptr; + } + } +} + + //- Calculate a reserve size (eg, doubling) based on the request length //- and the current capacity template<int SizeMin, int Numerator, class IntType> -inline IntType reserve_size(IntType requested, IntType capacity) +inline IntType reserve_size(IntType requested, IntType capacity) noexcept { static_assert(Numerator > 1, "Invalid numerator"); @@ -117,7 +276,7 @@ inline IntType reserve_size(IntType requested, IntType capacity) //- Calculate a reserve size based on the request length //- and the current capacity template<int SizeMin, int Numerator, int Denominator, class IntType> -inline IntType reserve_size(IntType requested, IntType capacity) +inline IntType reserve_size(IntType requested, IntType capacity) noexcept { static_assert(Numerator > Denominator, "Invalid numerator"); static_assert(Denominator > 0, "Invalid denominator"); diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C index 02223f1bd79ca3136a8b77bad2e5a21105415c2a..b4e4f543b0b1df86819a63d2e0c35d79554fbd23 100644 --- a/src/OpenFOAM/global/argList/argList.C +++ b/src/OpenFOAM/global/argList/argList.C @@ -37,6 +37,7 @@ License #include "IOobject.H" #include "dynamicCode.H" #include "simpleObjectRegistry.H" +#include "MemoryPool.H" #include "sigFpe.H" #include "sigInt.H" #include "sigQuit.H" @@ -2181,6 +2182,9 @@ void Foam::argList::parse sigQuit::set(bannerEnabled()); sigSegv::set(bannerEnabled()); + // Create memory pool (if any) after MPI has been setup + MemoryPool::create(bannerEnabled()); + if (UPstream::master() && bannerEnabled()) { Info<< "fileModificationChecking : " diff --git a/src/OpenFOAM/matrices/Matrix/Matrix.C b/src/OpenFOAM/matrices/Matrix/Matrix.C index bb1251cf62feadec74ae66583e25f0d38811dab6..a1652c1e7db1738a50fd288c998facd52d165835 100644 --- a/src/OpenFOAM/matrices/Matrix/Matrix.C +++ b/src/OpenFOAM/matrices/Matrix/Matrix.C @@ -244,7 +244,8 @@ inline Foam::Matrix<Form, Type>::Matrix template<class Form, class Type> Foam::Matrix<Form, Type>::~Matrix() { - delete[] v_; + // Accurate alignment information? + ListPolicy::deallocate(this->v_, (mRows_*nCols_)); } @@ -253,12 +254,10 @@ Foam::Matrix<Form, Type>::~Matrix() template<class Form, class Type> void Foam::Matrix<Form, Type>::clear() { - if (v_) - { - delete[] v_; - v_ = nullptr; - } + // Accurate alignment information? + ListPolicy::deallocate(this->v_, (mRows_*nCols_)); + v_ = nullptr; mRows_ = 0; nCols_ = 0; } diff --git a/src/OpenFOAM/matrices/Matrix/MatrixI.H b/src/OpenFOAM/matrices/Matrix/MatrixI.H index e16f30b694d62fe3e0252019aa177727186f03d7..d6ad74f77f77482545bc0bfa72c27f05f2a7efe4 100644 --- a/src/OpenFOAM/matrices/Matrix/MatrixI.H +++ b/src/OpenFOAM/matrices/Matrix/MatrixI.H @@ -38,7 +38,8 @@ inline void Foam::Matrix<Form, Type>::doAlloc() if (len > 0) { // With sign-check to avoid spurious -Walloc-size-larger-than - v_ = new Type[len]; + this->v_ = ListPolicy::allocate<Type>(len); + } } diff --git a/src/OpenFOAM/memory/pool/MemoryPool.H b/src/OpenFOAM/memory/pool/MemoryPool.H new file mode 100644 index 0000000000000000000000000000000000000000..56879c3d42ff58404911e56b0ed17bb6fb339bd6 --- /dev/null +++ b/src/OpenFOAM/memory/pool/MemoryPool.H @@ -0,0 +1,116 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2025 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +Class + Foam::MemoryPool + +Description + Optional memory management using a memory pool such as Umpire + (https://github.com/LLNL/Umpire). + + When compiled with Umpire, its use can be controlled by the + \c FOAM_MEMORY_POOL environment variable, or the + \c memory_pool Optimisation switch (etc/controlDict). + + It currently looks for any of the following entries, in this order: + - true - same as \em "host" + - false/none - disabled. + - \em "host" - uses host memory pool + - \em "system" - same as \em "host" + - \em "device" - uses device memory pool + - \em "managed" - uses managed host/device memory pool + . + + The parameters "size=nn" and "incr=nn" (in MegaBytes) can be used + to specify alternatives to the default sizing. + +\*---------------------------------------------------------------------------*/ + +#ifndef Foam_MemoryPool_H +#define Foam_MemoryPool_H + +#include <cstdint> // For size_t + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + +/*---------------------------------------------------------------------------*\ + Class MemoryPool Declaration +\*---------------------------------------------------------------------------*/ + +class MemoryPool +{ +public: + + // Constructors + + //- Create a memory pool instance (if not already active). + // Uses environment or etc/controlDict entry + static bool create(bool verbose = false); + + + // Destructor + + //- Remove the memory pool instance (currently does nothing) + static void destroy(bool verbose = false); + + + // Member Functions + + //- True if pool is active (ie, created and not suspended) + static bool active() noexcept; + + //- Suspend use of memory pool (for allocation). + // \return previous suspend status + static bool suspend() noexcept; + + //- Resume use of memory pool (if previously active) + static void resume() noexcept; + + //- Test if given pointer belongs to the pool + static bool is_pool(void *ptr); + + //- Allocate from pool (if active). + // \returns nullptr if the pool is not active + static void* try_allocate(std::size_t nbytes); + + //- Deallocate a pointer managed by the pool + // \returns True if a nullptr (no-op) or when the pointer was + // managed by the pool. + static bool try_deallocate(void *ptr); +}; + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace Foam + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#endif + +// ************************************************************************* //