ENH: improve OFstream append behaviour (#3160)

- previous support for file appending (unused in the meantime)
  specified opening with `std::ios_base::app`. However, this also
  enforces append behaviour for each write operation and thus
  disallows any seek/repositioning within the output file.

  Now treat append as an "append-like" behaviour instead.

  If the file already exists, its contents will be preserved and
  the *initial* output position is moved to the file end. All
  subsequent write operations respect the current file position
  without an additional seek-to-end on each write. This enables
  support of file appending/overwriting, but does preclude concurrent
  file output.

applications/test/OFstream/Test-OFstream.cxx

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -32,10 +32,14 @@ Description
 #include "IOstreams.H"
 #include "OSspecific.H"
 #include "argList.H"
+#include "clock.H"
+#include "Switch.H"
 #include "ListOps.H"
 
 using namespace Foam;
 
+std::string time_stamp;
+
 void listFiles(const fileName& dir)
 {
     wordList files = ListOps::create<word>
@@ -55,6 +59,41 @@ void listFiles(const fileName& dir)
 }
 
 
+OSstream& printInfo(OFstream& os)
+{
+    InfoErr
+        << "open: " << os.name() << nl
+        << "is_append: " << Switch::name(os.is_append())
+        << " tellp: "<< os.stdStream().tellp() << nl;
+
+    return InfoErr.stream();
+}
+
+
+void withHeader(OFstream& os)
+{
+    const auto tellp = os.stdStream().tellp();
+
+    if (tellp == 0)
+    {
+        InfoErr
+            << "Add header" << nl;
+        os  << "HEADER: " << time_stamp.c_str() << nl;
+    }
+}
+
+
+template<class OSstreamType>
+void generateLines(OSstreamType& os, label count = 1)
+{
+    for (label line = 1; line <= count; ++line)
+    {
+        os  << "[" << line
+            << "] =============================================" << nl;
+    }
+}
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 // Main program:
 
@@ -62,18 +101,31 @@ int main(int argc, char *argv[])
 {
     argList::addBoolOption("gz", "Use compression");
     argList::addBoolOption("append", "Use append mode");
+    argList::addBoolOption("seekend", "Seek to end after non-append open");
+    argList::addOption("seek", "value", "Seek from start (default: 100)");
     argList::addBoolOption("atomic", "Use atomic");
     argList::addBoolOption("keep", "Do not remove test directory");
     argList::addOption("write", "file", "test writing to file");
 
     #include "setRootCase.H"
 
+    // Same time-stamp for all generated files
+    time_stamp = clock::dateTime();
+
     const fileName baseDir("Test-OFstream-directory");
 
     Foam::mkDir(baseDir);
 
     InfoErr<< "mkdir: " << baseDir << endl;
 
+    Info<< "start:" << nl;
+    listFiles(baseDir);
+
+    const bool with_append = args.found("append");
+    const bool with_seekend = args.found("seekend");
+
+    const int seek_out = args.getOrDefault<int>("seek", 100);
+
     IOstreamOption streamOpt;
 
     if (args.found("gz"))
@@ -97,7 +149,6 @@ int main(int argc, char *argv[])
     {
         OFstream(baseDir/"dummy")() << "Some file content" << endl;
 
-        Foam::ln("dummy", baseDir/"Test2.txt");
         Foam::ln("dummy", baseDir/"Test3.txt");
         Foam::ln("dummy", baseDir/"Test4.txt");
         Foam::ln("dummy", baseDir/"Test4.txt.gz");
@@ -114,16 +165,58 @@ int main(int argc, char *argv[])
             append
         );
 
-        os << "=========================" << endl;
+        if (with_seekend && !with_append)
+        {
+            os.stdStream().seekp(0, std::ios_base::end);
+        }
+
+        printInfo(os);
+
+        withHeader(os);
 
-        InfoErr<< "open: " << os.name() << endl;
-        InfoErr<< "... sleep" << endl;
+        {
+            InfoErr<< "... seekp(" << seek_out << ")" << nl;
+
+            auto& oss = os.stdStream();
+
+            // Actually std::streampos, but cannot increment that
+
+            int64_t pos(seek_out);
+
+            const int64_t tellp_end = oss.tellp();
+
+            if (pos >= 0 && pos < tellp_end)
+            {
+                InfoErr
+                    << "... fill from " << label(pos)
+                    << " to " << label(tellp_end) << nl;
+
+                oss.seekp(pos);
+
+                while (pos < tellp_end)
+                {
+                    // Fill with char 'X', rely on streambuf buffering
+                    oss << 'X';
+                    ++pos;
+                }
+
+                oss.seekp(seek_out);
+                os << "More content [at " << seek_out << ']' << endl;
+            }
+        }
+
+        generateLines(os, 4);
+
+        printInfo(os)
+            << "... sleep" << endl;
 
         listFiles(baseDir);
 
         sleep(2);
 
-        os << "+++++++++++++++++++++++++++++++++++" << endl;
+        os << "[new content] +++++++++++++++++++++++++++++++++++" << endl;
+
+
     }
 
     {
@@ -135,16 +228,24 @@ int main(int argc, char *argv[])
             // NON_APPEND
         );
 
-        os << "=========================" << endl;
+        if (with_seekend)  // NON_APPEND
+        {
+            os.stdStream().seekp(0, std::ios_base::end);
+        }
+        printInfo(os);
 
-        InfoErr<< "open: " << os.name() << endl;
-        InfoErr<< "... sleep" << endl;
+        withHeader(os);
+
+        generateLines(os, 4);
+
+        printInfo(os)
+            << "... sleep" << endl;
 
         listFiles(baseDir);
 
         sleep(2);
 
-        os << "+++++++++++++++++++++++++++++++++++" << endl;
+        os << "[new content] +++++++++++++++++++++++++++++++++++" << endl;
     }
     {
         OFstream os
@@ -155,16 +256,26 @@ int main(int argc, char *argv[])
             IOstreamOption::APPEND
         );
 
-        os << "=========================" << endl;
+        // No seekend with APPEND, but try anyhow
+        if (with_seekend)
+        {
+            os.stdStream().seekp(0, std::ios_base::end);
+        }
+
+        printInfo(os);
 
-        InfoErr<< "open: " << os.name() << endl;
-        InfoErr<< "... sleep" << endl;
+        withHeader(os);
+
+        generateLines(os, 4);
+
+        printInfo(os)
+            << "... sleep" << endl;
 
         listFiles(baseDir);
 
         sleep(2);
 
-        os << "+++++++++++++++++++++++++++++++++++" << endl;
+        os << "[new content] +++++++++++++++++++++++++++++++++++" << endl;
     }
     {
         OFstream os
@@ -174,10 +285,16 @@ int main(int argc, char *argv[])
             IOstreamOption::COMPRESSED
         );
 
-        os << "=========================" << endl;
+        // No seekend with COMPRESSED
 
-        InfoErr<< "open: " << os.name() << endl;
-        InfoErr<< "... sleep" << endl;
+        printInfo(os);
+
+        withHeader(os);
+
+        generateLines(os, 4);
+
+        printInfo(os)
+            << "... sleep" << endl;
 
         listFiles(baseDir);
 
@@ -193,10 +310,19 @@ int main(int argc, char *argv[])
             // ASCII UNCOMPRESSED NON_APPEND
         );
 
-        os << "=========================" << endl;
+        if (with_seekend)  // NON_APPEND
+        {
+            os.stdStream().seekp(0, std::ios_base::end);
+        }
+
+        printInfo(os);
+
+        withHeader(os);
+
+        generateLines(os, 4);
 
-        InfoErr<< "open: " << os.name() << endl;
-        InfoErr<< "... sleep" << endl;
+        printInfo(os)
+            << "... sleep" << endl;
 
         listFiles(baseDir);
 

src/OpenFOAM/db/IOstreams/Fstreams/OFstream.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2017-2023 OpenCFD Ltd.
+    Copyright (C) 2017-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -28,7 +28,22 @@ Class
     Foam::OFstream
 
 Description
-    Output to file stream, using an OSstream
+    Output to file stream as an OSstream, normally using std::ofstream
+    for the actual output.
+
+Note
+    The atomic output works by creating an intermediate temporary file,
+    which is renamed as an atomic operation when closing. It is not
+    possible, or particularly desirable, to have an atomic in combination
+    with append behaviour. If both are specified, append has priority.
+
+    When an output file is opened in \c append mode, it has "append-like"
+    behaviour, which means that existing file content will be preserved
+    and the \em initial output position is moved to the file end.
+
+    Subsequent write operations respect the current file position without an
+    additional seek-to-end on each write. This enables support of file
+    overwriting, but precludes concurrent file output.
 
 SourceFiles
     OFstream.C
@@ -64,12 +79,11 @@ public:
 
     // Constructors
 
-        //- Construct a null output file stream.
-        //  Behaves like \c /dev/null and is named accordingly
+        //- Construct a null output file stream that behaves like \c /dev/null
         explicit OFstream(std::nullptr_t);
 
         //- Construct with specified atomic behaviour
-        //- from pathname, stream option, optional append
+        //- from pathname, stream option, optional append (see note).
         OFstream
         (
             IOstreamOption::atomicType atomic,
@@ -78,7 +92,8 @@ public:
             IOstreamOption::appendType append = IOstreamOption::NON_APPEND
         );
 
-        //- Construct from pathname and other specifications
+        //- Construct from pathname and other specifications.
+        //  See note on append mode.
         explicit OFstream
         (
             const fileName& pathname,
@@ -89,7 +104,8 @@ public:
             OFstream(IOstreamOption::NON_ATOMIC, pathname, streamOpt, append)
         {}
 
-        //- Construct from pathname, format (uncompressed), optional append,
+        //- Construct from pathname, format (uncompressed),
+        //- optional append (see note),
         //- atomic behaviour as per system default
         OFstream
         (
@@ -103,7 +119,8 @@ public:
         {}
 
         //- Construct with specified atomic behaviour
-        //- from pathname, format (uncompressed), optional append
+        //- from pathname, format (uncompressed),
+        //- optional append (see note).
         OFstream
         (
             IOstreamOption::atomicType atomic,
@@ -141,6 +158,15 @@ public:
         virtual void rewind();
 
 
+    // Output stream modes
+
+        //- True if file already existed \em and was opened in append-like mode
+        bool is_append() const noexcept { return ofstreamPointer::is_append(); }
+
+        //- True if file creation behaves as atomic
+        bool is_atomic() const noexcept { return ofstreamPointer::is_atomic(); }
+
+
     // Print
 
         //- Print stream description

src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointer.H

@@ -194,20 +194,31 @@ public:
 
 class ofstreamPointer
 {
+    // Private Data Types
+
+        //- The file open/creation type (bitmask)
+        enum modeType : char
+        {
+            NONE = 0,
+            ATOMIC = 0x1,
+            APPEND = 0x2
+        };
+
+
     // Private Data
 
         //- The stream pointer (ofstream | ogzstream | ocountstream, ...)
         std::unique_ptr<std::ostream> ptr_;
 
-        //- Atomic file creation
-        bool atomic_;
+        //- File output/creation type (atomic, append etc)
+        char mode_;
 
 
 protected:
 
     // Protected Member Functions
 
-        //- Reopen for compressed/non-compressed
+        //- Reopen for compressed/non-compressed. Discards append status.
         void reopen(const std::string& pathname);
 
         //- Close stream and rename file
@@ -249,12 +260,17 @@ public:
         //  \param atomic   Write into temporary file (not target file).
         //      This option should only be used with a stream wrapper
         //      (eg, OFstream) that handles the final renaming.
+        //
+        //  \note The append mode is actually 'append-like', which means
+        //        that it preserves existing files and moves the \em initial
+        //        position to the end, but does not enforce seeking to the end
+        //        for every write operation.
         explicit ofstreamPointer
         (
             const fileName& pathname,
             IOstreamOption streamOpt = IOstreamOption(),
-            const bool append = false,
-            const bool atomic = false
+            bool append = false,
+            bool atomic = false
         );
 
         //- Construct from pathname, compression, append, file handling atomic
@@ -268,12 +284,12 @@ public:
         (
             const fileName& pathname,
             IOstreamOption::compressionType comp,
-            const bool append = false,
-            const bool atomic = false
+            bool append = false,
+            bool atomic = false
         );
 
 
-    // Member Functions
+    // Static Functions
 
         //- True if compiled with libz support
         static bool supports_gz() noexcept;
@@ -293,13 +309,21 @@ public:
         //- Which compression type?
         IOstreamOption::compressionType whichCompression() const;
 
+        //- True if file already existed \em and was opened in append-like mode
+        bool is_append() const noexcept { return (mode_ & modeType::APPEND); }
+
+        //- True if file creation behaves as atomic
+        bool is_atomic() const noexcept { return (mode_ & modeType::ATOMIC); }
+
 
     // Edit
 
-        //- Return managed pointer and release ownership
+        //- Return managed pointer and release ownership.
+        //  Likely invalidates is_append(), is_atomic().
         std::ostream* release() noexcept { return ptr_.release(); }
 
         //- Replace the managed pointer
+        //  Likely invalidates is_append(), is_atomic().
         void reset(std::ostream* ptr) noexcept { ptr_.reset(ptr); }
 
 

src/OpenFOAM/db/IOstreams/Fstreams/fstreamPointers.C

@@ -88,14 +88,14 @@ Foam::ifstreamPointer::ifstreamPointer
 Foam::ofstreamPointer::ofstreamPointer() noexcept
 :
     ptr_(),
-    atomic_(false)
+    mode_(modeType::NONE)
 {}
 
 
 Foam::ofstreamPointer::ofstreamPointer(std::nullptr_t)
 :
     ptr_(new Foam::ocountstream),
-    atomic_(false)
+    mode_(modeType::NONE)
 {}
 
 
@@ -103,27 +103,33 @@ Foam::ofstreamPointer::ofstreamPointer
 (
     const fileName& pathname,
     IOstreamOption streamOpt,
-    const bool append,
-    const bool atomic
+    bool append,
+    bool atomic
 )
 :
     ptr_(),
-    atomic_(atomic)
+    mode_(modeType::NONE)
 {
-    std::ios_base::openmode mode
+    // Leave std::ios_base::trunc implicitly handled (easier for append mode)
+
+    std::ios_base::openmode openmode
     (
         std::ios_base::out | std::ios_base::binary
     );
 
     if (append)
     {
-        mode |= std::ios_base::app;
+        // Handle an "append-like" mode by opening "r+b" and NOT as "ab"
+        // - file already exists: Sets read position to start
+        // - file does not exist: Error
+
+        openmode |= std::ios_base::in;  // [SIC] - use read bit, not append!
 
         // Cannot append to gzstream
         streamOpt.compression(IOstreamOption::UNCOMPRESSED);
 
         // Cannot use append + atomic operation, without lots of extra work
-        atomic_ = false;
+        atomic = false;
     }
 
 
@@ -146,9 +152,9 @@ Foam::ofstreamPointer::ofstreamPointer
 
         #ifdef HAVE_LIBZ
         // TBD:
-        // atomic_ = true;  // Always treat COMPRESSED like an atomic
+        // atomic = true;  // Always treat COMPRESSED like an atomic
 
-        const fileName& target = (atomic_ ? pathname_tmp : pathname_gz);
+        const fileName& target = (atomic ? pathname_tmp : pathname_gz);
 
         // Remove old uncompressed version (if any)
         fType = Foam::type(pathname, false);
@@ -158,7 +164,7 @@ Foam::ofstreamPointer::ofstreamPointer
         }
 
         // Avoid writing into symlinked files (non-append mode)
-        if (!append || atomic_)
+        if (!append || atomic)
         {
             fType = Foam::type(target, false);
             if (fType == fileName::SYMLINK)
@@ -167,7 +173,7 @@ Foam::ofstreamPointer::ofstreamPointer
             }
         }
 
-        ptr_.reset(new ogzstream(target, mode));
+        ptr_.reset(new ogzstream(target, openmode));
 
         #else /* HAVE_LIBZ */
 
@@ -184,7 +190,7 @@ Foam::ofstreamPointer::ofstreamPointer
 
     if (IOstreamOption::COMPRESSED != streamOpt.compression())
     {
-        const fileName& target = (atomic_ ? pathname_tmp : pathname);
+        const fileName& target = (atomic ? pathname_tmp : pathname);
 
         // Remove old compressed version (if any)
         fType = Foam::type(pathname_gz, false);
@@ -194,7 +200,7 @@ Foam::ofstreamPointer::ofstreamPointer
         }
 
         // Avoid writing into symlinked files (non-append mode)
-        if (!append || atomic_)
+        if (!append || atomic)
         {
             fType = Foam::type(target, false);
             if (fType == fileName::SYMLINK)
@@ -203,7 +209,49 @@ Foam::ofstreamPointer::ofstreamPointer
             }
         }
 
-        ptr_.reset(new std::ofstream(target, mode));
+        // File pointer (std::ofstream)
+        auto filePtr = std::make_unique<std::ofstream>(target, openmode);
+
+        // Continue handling for pseudo-append mode (always non-atomic)
+        if (append)
+        {
+            if (filePtr->good())
+            {
+                // Success if file already exists.
+                // Set output position to the end - like std::ios_base::ate
+
+                filePtr->seekp(0, std::ios_base::end);
+            }
+            else
+            {
+                // Error if file does not already exist.
+                // Reopen as regular output mode only.
+
+                append = false;  // Did not open an existing file
+
+                if (filePtr->is_open())
+                {
+                    filePtr->close();
+                }
+                filePtr->clear();
+                filePtr->open
+                (
+                    target,
+                    (std::ios_base::out | std::ios_base::binary)
+                );
+            }
+        }
+        ptr_.reset(filePtr.release());
+    }
+
+    // Remember the settings actually used
+    if (append)
+    {
+        mode_ |= modeType::APPEND;
+    }
+    if (atomic)
+    {
+        mode_ |= modeType::ATOMIC;
     }
 }
 
@@ -212,8 +260,8 @@ Foam::ofstreamPointer::ofstreamPointer
 (
     const fileName& pathname,
     IOstreamOption::compressionType comp,
-    const bool append,
-    const bool atomic
+    bool append,
+    bool atomic
 )
 :
     ofstreamPointer
@@ -237,12 +285,12 @@ void Foam::ifstreamPointer::open
     // Forcibly close old stream (if any)
     ptr_.reset(nullptr);
 
-    const std::ios_base::openmode mode
+    const std::ios_base::openmode openmode
     (
         std::ios_base::in | std::ios_base::binary
     );
 
-    ptr_.reset(new std::ifstream(pathname, mode));
+    ptr_.reset(new std::ifstream(pathname, openmode));
 
     if (!ptr_->good())
     {
@@ -254,7 +302,7 @@ void Foam::ifstreamPointer::open
         {
             #ifdef HAVE_LIBZ
 
-            ptr_.reset(new igzstream(pathname_gz, mode));
+            ptr_.reset(new igzstream(pathname_gz, openmode));
 
             #else /* HAVE_LIBZ */
 
@@ -311,7 +359,7 @@ void Foam::ofstreamPointer::reopen(const std::string& pathname)
         gz->close();
         gz->clear();
 
-        if (atomic_)
+        if (mode_ & modeType::ATOMIC)
         {
             gz->open
             (
@@ -341,10 +389,12 @@ void Foam::ofstreamPointer::reopen(const std::string& pathname)
         }
         file->clear();
 
-        // Don't need original request to append since rewind implies
-        // trashing that anyhow.
+        // Discard opened in append-like mode and file already existed
+        // information since rewind implies trashing that anyhow.
+
+        mode_ &= ~modeType::APPEND;
 
-        if (atomic_)
+        if (mode_ & modeType::ATOMIC)
         {
             file->open
             (
@@ -367,7 +417,10 @@ void Foam::ofstreamPointer::reopen(const std::string& pathname)
 
 void Foam::ofstreamPointer::close(const std::string& pathname)
 {
-    if (!atomic_ || pathname.empty()) return;
+    if (pathname.empty() || !(mode_ & modeType::ATOMIC))
+    {
+        return;
+    }
 
     #ifdef HAVE_LIBZ
     auto* gz = dynamic_cast<ogzstream*>(ptr_.get());