Commit 6e2b7be9 authored by Mark Olesen's avatar Mark Olesen

ENH: direct access to wrapped ifstream/ofstream with compression (#1805)

- previously hidden as Detail::[IO]FstreamAllocator, now exposed
  directly as [io]fstreamPointer, which allows reuse for
  std::ifstream, std::ofstream wrapping, without the additional
  ISstream, OSstream layers.

  These stream pointers have some characteristics similar to a
  unique_ptr.

- restrict direct gzstream usage to two files (fstreamPointers.C,
  gzstream.C) which improves localization and makes it simpler to
  enable/disable with the `HAVE_LIBZ` define.

  The HAVE_LIBZ define is currently simply hard-coded in the
  Make/options.

  If compiled WITHOUT libz support:
    - reading gz files : FatalError
    - writing gz files : emit warning and downgrade to uncompressed
    - warn if compression is specified in the case controlDict
      and downgrade to uncompressed

ENH: minor updates to gzstream interface for C++11

- support construct/open with std::string for the file names.

CONFIG: provisioning for have_libz detection as wmake/script
parent b2bded48
Test-fstreamPointer.C
EXE = $(FOAM_USER_APPBIN)/Test-fstreamPointer
/* EXE_INC = */
/* EXE_LIBS = */
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2020 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
Application
Test-fstreamPointer
Description
Low-level fstream tests
\*---------------------------------------------------------------------------*/
#include "argList.H"
#include "refPtr.H"
#include "fstreamPointer.H"
using namespace Foam;
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Main program:
int main(int argc, char *argv[])
{
argList::noBanner();
argList::noParallel();
argList::addOption
(
"output",
"file",
"Output file name for cat"
);
argList::addArgument("file1");
argList::addArgument("...");
argList::addArgument("fileN");
argList::noMandatoryArgs();
#include "setRootCase.H"
if (args.size() <= 1)
{
InfoErr<< "\nNo input files specified .. stopping\n" << endl;
return 0;
}
refPtr<std::ostream> osRef;
fileName outputName;
if (args.readIfPresent("output", outputName))
{
InfoErr<< "output: " << outputName;
IOstreamOption::compressionType comp(IOstreamOption::UNCOMPRESSED);
if (outputName.hasExt("gz"))
{
comp = IOstreamOption::COMPRESSED;
outputName.removeExt();
InfoErr<< " [compress]";
}
InfoErr<< nl;
osRef.reset(ofstreamPointer(outputName, comp).release());
}
else
{
osRef.ref(std::cout);
InfoErr<< "output: stdout" << nl;
}
auto& os = osRef.ref();
for (label argi = 1; argi < args.size(); ++argi)
{
const fileName inputName(args[argi]);
InfoErr<< "input: " << inputName;
ifstreamPointer isPtr(inputName);
if (!isPtr.get() || !isPtr->good())
{
InfoErr<< " (not good)" << nl;
continue;
}
InfoErr<< nl;
auto& is = *isPtr;
// Loop getting single characters
// - not efficient, but that is not the test here anyhow
char c;
while (is.get(c))
{
os << c;
}
}
InfoErr<< "\nEnd\n" << endl;
return 0;
}
// ************************************************************************* //
......@@ -238,6 +238,7 @@ $(memstream)/ListStream.C
Fstreams = $(Streams)/Fstreams
$(Fstreams)/IFstream.C
$(Fstreams)/OFstream.C
$(Fstreams)/fstreamPointers.C
$(Fstreams)/masterOFstream.C
Tstreams = $(Streams)/Tstreams
......
......@@ -13,5 +13,7 @@ else
LIB_LIBS += -L$(FOAM_LIBBIN)/dummy -lPstream
endif
LIB_LIBS += \
-lz
/* libz */
EXE_INC += -DHAVE_LIBZ
LIB_LIBS += -lz
......@@ -5,35 +5,20 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017 OpenCFD Ltd.
Copyright (C) 2017-2020 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
InClass
Foam::Fstream
This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
Description
Input/output from file streams.
File stream includes.
\*---------------------------------------------------------------------------*/
#ifndef Fstream_H
#define Fstream_H
#include "fstreamPointer.H"
#include "IFstream.H"
#include "OFstream.H"
......
......@@ -28,7 +28,6 @@ License
#include "IFstream.H"
#include "OSspecific.H"
#include "gzstream.h"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
......@@ -38,43 +37,6 @@ namespace Foam
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::Detail::IFstreamAllocator::IFstreamAllocator(const fileName& pathname)
:
allocatedPtr_(nullptr),
detectedCompression_(IOstream::UNCOMPRESSED)
{
if (pathname.empty())
{
if (IFstream::debug)
{
InfoInFunction << "Cannot open null file " << endl;
}
}
const std::ios_base::openmode mode(std::ios_base::in|std::ios_base::binary);
allocatedPtr_.reset(new std::ifstream(pathname, mode));
// If the file is compressed, decompress it before reading.
if (!allocatedPtr_->good() && isFile(pathname + ".gz", false))
{
if (IFstream::debug)
{
InfoInFunction << "Decompressing " << pathname + ".gz" << endl;
}
allocatedPtr_.reset(new igzstream((pathname + ".gz").c_str(), mode));
if (allocatedPtr_->good())
{
detectedCompression_ = IOstream::COMPRESSED;
}
}
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::IFstream::IFstream
......@@ -83,32 +45,47 @@ Foam::IFstream::IFstream
IOstreamOption streamOpt
)
:
Detail::IFstreamAllocator(pathname),
ISstream(*allocatedPtr_, pathname, streamOpt)
Foam::ifstreamPointer(pathname),
ISstream(*(ifstreamPointer::get()), pathname, streamOpt)
{
IOstream::compression(IFstreamAllocator::detectedCompression_);
IOstream::compression(ifstreamPointer::whichCompression());
setClosed();
setState(allocatedPtr_->rdstate());
setState(ifstreamPointer::get()->rdstate());
if (!good())
if (good())
{
if (debug)
{
InfoInFunction
<< "Could not open file " << pathname
<< " for input" << nl << info() << Foam::endl;
}
setBad();
setOpened();
}
else
{
setOpened();
setBad();
}
lineNumber_ = 1;
if (debug)
{
if (pathname.empty())
{
InfoInFunction
<< "Cannot open empty file name"
<< Foam::endl;
}
else if (IOstreamOption::COMPRESSED == IOstream::compression())
{
InfoInFunction
<< "Decompressing " << (this->name() + ".gz") << Foam::endl;
}
if (!opened())
{
InfoInFunction
<< "Could not open file " << pathname
<< " for input\n" << info() << Foam::endl;
}
}
}
......@@ -116,51 +93,44 @@ Foam::IFstream::IFstream
std::istream& Foam::IFstream::stdStream()
{
if (!allocatedPtr_)
std::istream* ptr = ifstreamPointer::get();
if (!ptr)
{
FatalErrorInFunction
<< "No stream allocated"
<< "No stream allocated\n"
<< abort(FatalError);
}
return *allocatedPtr_;
return *ptr;
}
const std::istream& Foam::IFstream::stdStream() const
{
if (!allocatedPtr_)
const std::istream* ptr = ifstreamPointer::get();
if (!ptr)
{
FatalErrorInFunction
<< "No stream allocated"
<< "No stream allocated\n"
<< abort(FatalError);
}
return *allocatedPtr_;
return *ptr;
}
void Foam::IFstream::rewind()
{
lineNumber_ = 1; // Reset line number
igzstream* gzPtr = nullptr;
try
{
gzPtr = dynamic_cast<igzstream*>(allocatedPtr_.get());
}
catch (const std::bad_cast&)
{
gzPtr = nullptr;
}
lineNumber_ = 1; // Reset line number
if (gzPtr)
if (IOstreamOption::COMPRESSED == ifstreamPointer::whichCompression())
{
// Need special treatment for gzstream.
gzPtr->close();
gzPtr->clear();
gzPtr->open((this->name() + ".gz").c_str());
// Special treatment for compressed stream
ifstreamPointer::reopen_gz(this->name() + ".gz");
setState(gzPtr->rdstate());
setState(ifstreamPointer::get()->rdstate());
}
else
{
......
......@@ -39,53 +39,21 @@ SourceFiles
#define IFstream_H
#include "ISstream.H"
#include "fileName.H"
#include "className.H"
#include <fstream>
#include <memory>
#include "fstreamPointer.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
namespace Detail
{
/*---------------------------------------------------------------------------*\
Class Detail::IFstreamAllocator Declaration
\*---------------------------------------------------------------------------*/
//- A std::istream with the ability to handle compressed files
class IFstreamAllocator
{
protected:
// Member Data
//- The allocated stream pointer (ifstream or igzstream).
std::unique_ptr<std::istream> allocatedPtr_;
//- The detected compression type
IOstream::compressionType detectedCompression_;
// Constructors
//- Construct from pathname
IFstreamAllocator(const fileName& pathname);
};
} // End namespace Detail
/*---------------------------------------------------------------------------*\
Class IFstream Declaration
\*---------------------------------------------------------------------------*/
class IFstream
:
public Detail::IFstreamAllocator,
private Foam::ifstreamPointer,
public ISstream
{
public:
......@@ -107,8 +75,8 @@ public:
IFstream
(
const fileName& pathname,
streamFormat fmt,
versionNumber ver = currentVersion
IOstreamOption::streamFormat fmt,
IOstreamOption::versionNumber ver = currentVersion
)
:
IFstream(pathname, IOstreamOption(fmt, ver))
......@@ -121,7 +89,8 @@ public:
// Member Functions
// Access
//- Get character(s)
using ISstream::get;
//- Read/write access to the name of the stream
using ISstream::name;
......@@ -141,7 +110,7 @@ public:
// Print
//- Print stream description to Ostream
//- Print stream description
virtual void print(Ostream& os) const;
......
......@@ -28,7 +28,6 @@ License
#include "OFstream.H"
#include "OSspecific.H"
#include "gzstream.h"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
......@@ -38,106 +37,48 @@ namespace Foam
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::Detail::OFstreamAllocator::OFstreamAllocator
Foam::OFstream::OFstream
(
const fileName& pathname,
IOstream::compressionType comp,
IOstreamOption streamOpt,
const bool append
)
:
allocatedPtr_(nullptr)
Foam::ofstreamPointer(pathname, streamOpt.compression(), append),
OSstream(*(ofstreamPointer::get()), pathname, streamOpt)
{
if (pathname.empty())
{
if (OFstream::debug)
{
InfoInFunction << "Cannot open null file " << endl;
}
}
setClosed();
setState(ofstreamPointer::get()->rdstate());
std::ios_base::openmode mode(std::ios_base::out|std::ios_base::binary);
if (append)
if (good())
{
mode |= std::ios_base::app;
setOpened();
}
if (comp == IOstream::COMPRESSED)
else
{
// Get identically named uncompressed version out of the way
fileName gzPathName(pathname + ".gz");
fileName::Type pathType = Foam::type(pathname, false);
if (pathType == fileName::FILE || pathType == fileName::LINK)
{
rm(pathname);
}
setBad();
}
if (!append && Foam::type(gzPathName) == fileName::LINK)
{
// Disallow writing into softlink to avoid any problems with
// e.g. softlinked initial fields
rm(gzPathName);
}
lineNumber_ = 1;
allocatedPtr_.reset(new ogzstream(gzPathName.c_str(), mode));
}
else
if (debug)
{
// Get identically named compressed version out of the way
fileName gzPathName(pathname + ".gz");
fileName::Type gzType = Foam::type(gzPathName, false);
if (gzType == fileName::FILE || gzType == fileName::LINK)
if (pathname.empty())
{
rm(gzPathName);
InfoInFunction
<< "Cannot open empty file name"
<< Foam::endl;
}
if (!append && Foam::type(pathname, false) == fileName::LINK)
{
// Disallow writing into softlink to avoid any problems with
// e.g. softlinked initial fields
rm(pathname);
}
allocatedPtr_.reset(new std::ofstream(pathname, mode));
}
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::OFstream::OFstream
(
const fileName& pathname,
IOstreamOption streamOpt,
const bool append
)
:
Detail::OFstreamAllocator(pathname, streamOpt.compression(), append),
OSstream(*allocatedPtr_, pathname, streamOpt)
{
setClosed();
setState(allocatedPtr_->rdstate());
if (!good())
{
if (debug)
if (!opened())
{
InfoInFunction
<< "Could not open file " << pathname
<< " for output" << nl << info() << Foam::endl;
<< " for output\n" << info() << Foam::endl;
}
setBad();
}
else
{
setOpened();
}
lineNumber_ = 1;
}
......@@ -145,23 +86,31 @@ Foam::OFstream::OFstream
std::ostream& Foam::OFstream::stdStream()
{
if (!allocatedPtr_)
std::ostream* ptr = ofstreamPointer::get();
if (!ptr)
{
FatalErrorInFunction
<< "No stream allocated." << abort(FatalError);
<< "No stream allocated\n"
<< abort(FatalError);
}
return *allocatedPtr_;
return *ptr;
}
const std::ostream& Foam::OFstream::stdStream() const
{
if (!allocatedPtr_)
const std::ostream* ptr = ofstreamPointer::get();
if (!ptr)
{
FatalErrorInFunction
<< "No stream allocated." << abort(FatalError);
<< "No stream allocated\n"
<< abort(FatalError);
}
return *allocatedPtr_;
return *ptr;
}
......
......@@ -39,55 +39,21 @@ SourceFiles
#define OFstream_H
#include "OSstream.H"
#include "fileName.H"
#include "className.H"
#include <fstream>
#include <memory>
#include "fstreamPointer.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{