Commit c4de3e0a authored by Mark Olesen's avatar Mark Olesen
Browse files

ENH: enhancements to behaviour of token

- improved memory alignment reduces overhead for Int32 compilation

- added move/swap semantics

- made the type() readonly in favour of setVariant() to allow change
  of variant within a particular storage representation.
  Eg, STRING -> VERBATIMSTRING.
parent e1b71c02
Test-token.C
EXE = $(FOAM_USER_APPBIN)/Test-token
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2017 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Description
Test token construct assign etc.
\*---------------------------------------------------------------------------*/
#include "argList.H"
#include "IOobject.H"
#include "IOstreams.H"
#include "IFstream.H"
#include "StringStream.H"
#include "cpuTime.H"
#include "DynamicList.H"
using namespace Foam;
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Main program:
int main(int argc, char *argv[])
{
argList::noBanner();
argList::noParallel();
argList args(argc, argv, false, true);
token tok1;
Info<< "construct null: " << tok1.info() << endl;
tok1 = double(3.14159);
Info<< "assign double: " << tok1.info() << endl;
token tok2(tok1);
Info<< "copy construct: " << tok2.info() << endl;
tok1 = word("this-word");
Info<< "assign word: " << tok1.info() << endl;
token tok3(tok1);
Info<< "copy construct: " << tok3.info() << endl;
Info<< "orig: " << tok1.info() << endl;
token tok4(std::move(tok1));
Info<< "move construct: " << tok4.info() << endl;
Info<< "orig: " << tok1.info() << endl;
tok3 = tok4;
Info<< "assign token: " << tok3.info() << endl;
Info<< "orig: " << tok4.info() << endl;
return 0;
}
// ************************************************************************* //
......@@ -33,6 +33,7 @@ Description
#include "IFstream.H"
#include "StringStream.H"
#include "cpuTime.H"
#include "DynamicList.H"
using namespace Foam;
......@@ -69,6 +70,8 @@ int main(int argc, char *argv[])
IStringStream is(rawArg);
DynamicList<token> tokens;
while (is.good())
{
token tok(is);
......@@ -83,12 +86,23 @@ int main(int argc, char *argv[])
<< " lookahead: '" << char(lookahead) << "'"
<< endl;
}
if (tok.good())
{
tokens.append(std::move(tok));
if (verbose)
{
Info<< "after append: " << tok.info() << endl;
}
}
}
if (verbose)
{
Info<< nl;
IOobject::writeDivider(Info);
Info<< "tokenList:" << tokens << endl;
}
}
}
......
......@@ -27,7 +27,7 @@ License
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::Istream::putBack(const token& t)
void Foam::Istream::putBack(const token& tok)
{
if (bad())
{
......@@ -43,13 +43,13 @@ void Foam::Istream::putBack(const token& t)
}
else
{
putBackToken_ = t;
putBackToken_ = tok;
putBack_ = true;
}
}
bool Foam::Istream::getBack(token& t)
bool Foam::Istream::getBack(token& tok)
{
if (bad())
{
......@@ -59,7 +59,7 @@ bool Foam::Istream::getBack(token& t)
}
else if (putBack_)
{
t = putBackToken_;
tok = putBackToken_;
putBack_ = false;
return true;
}
......@@ -68,15 +68,15 @@ bool Foam::Istream::getBack(token& t)
}
bool Foam::Istream::peekBack(token& t)
bool Foam::Istream::peekBack(token& tok)
{
if (putBack_)
{
t = putBackToken_;
tok = putBackToken_;
}
else
{
t = token::undefinedToken;
tok = token::undefinedToken;
}
return putBack_;
......
......@@ -96,16 +96,16 @@ public:
//- Put back token
// Only a single put back is permitted
void putBack(const token&);
void putBack(const token& tok);
//- Get the put back token if there is one and return true.
// Return false if no put back token is available.
bool getBack(token&);
bool getBack(token& tok);
//- Peek at the put back token without removing it.
// Returns false if no put back token is available and set the
// token to undefined.
bool peekBack(token&);
bool peekBack(token& tok);
//- Return next token from stream
virtual Istream& read(token&) = 0;
......@@ -116,7 +116,7 @@ public:
//- Read a word
virtual Istream& read(word&) = 0;
// Read a string (including enclosing double-quotes)
//- Read a string (including enclosing double-quotes)
virtual Istream& read(string&) = 0;
//- Read a label
......
......@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
......@@ -164,22 +164,20 @@ Foam::Istream& Foam::UIPstream::read(token& t)
// Word
case token::tokenType::WORD :
{
word* pval = new word;
if (read(*pval))
word val;
if (read(val))
{
if (token::compound::isCompound(*pval))
if (token::compound::isCompound(val))
{
t = token::compound::New(*pval, *this).ptr();
delete pval;
t = token::compound::New(val, *this).ptr();
}
else
{
t = pval;
t = std::move(val);
}
}
else
{
delete pval;
t.setBad();
}
return *this;
......@@ -190,26 +188,25 @@ Foam::Istream& Foam::UIPstream::read(token& t)
{
// Recurse to read actual string
read(t);
t.type() = token::tokenType::VERBATIMSTRING;
t.setType(token::tokenType::VERBATIMSTRING);
return *this;
}
case token::tokenType::VARIABLE :
{
// Recurse to read actual string
read(t);
t.type() = token::tokenType::VARIABLE;
t.setType(token::tokenType::VARIABLE);
return *this;
}
case token::tokenType::STRING :
{
string* pval = new string;
if (read(*pval))
string val;
if (read(val))
{
t = pval;
t = std::move(val);
}
else
{
delete pval;
t.setBad();
}
return *this;
......
......@@ -116,21 +116,18 @@ char Foam::ISstream::nextValid()
void Foam::ISstream::readWordToken(token& t)
{
word* wPtr = new word;
if (read(*wPtr).bad())
word val;
if (read(val).bad())
{
delete wPtr;
t.setBad();
}
else if (token::compound::isCompound(*wPtr))
else if (token::compound::isCompound(val))
{
t = token::compound::New(*wPtr, *this).ptr();
delete wPtr;
t = token::compound::New(val, *this).ptr();
}
else
{
t = wPtr; // Token takes ownership
t = std::move(val); // Move contents to token
}
}
......@@ -192,16 +189,15 @@ Foam::Istream& Foam::ISstream::read(token& t)
case token::BEGIN_STRING :
{
putback(c);
string* sPtr = new string;
if (read(*sPtr).bad())
string val;
if (read(val).bad())
{
delete sPtr;
t.setBad();
}
else
{
t = sPtr; // Token takes ownership
t = std::move(val); // Move contents to token
}
return *this;
......@@ -219,17 +215,16 @@ Foam::Istream& Foam::ISstream::read(token& t)
else if (nextC == token::BEGIN_BLOCK)
{
// Verbatim string: #{ ... #}
string* sPtr = new string;
if (readVerbatim(*sPtr).bad())
string val;
if (readVerbatim(val).bad())
{
delete sPtr;
t.setBad();
}
else
{
t = sPtr; // Token takes ownership
t.type() = token::tokenType::VERBATIMSTRING;
t = std::move(val); // Move contents to token
t.setType(token::tokenType::VERBATIMSTRING);
}
}
else
......@@ -259,17 +254,15 @@ Foam::Istream& Foam::ISstream::read(token& t)
putback(nextC);
putback(c);
string* sPtr = new string;
if (readVariable(*sPtr).bad())
string val;
if (readVariable(val).bad())
{
delete sPtr;
t.setBad();
}
else
{
t = sPtr; // Token takes ownership
t.type() = token::tokenType::VARIABLE;
t = std::move(val); // Move contents to token
t.setType(token::tokenType::VARIABLE);
}
}
else
......
......@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
......@@ -29,14 +29,14 @@ License
namespace Foam
{
const char* const token::typeName = "token";
token token::undefinedToken;
typedef token::compound tokenCompound;
defineTypeNameAndDebug(tokenCompound, 0);
defineRunTimeSelectionTable(tokenCompound, Istream);
}
const char* const Foam::token::typeName = "token";
const Foam::token Foam::token::undefinedToken;
// * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * * //
......@@ -94,7 +94,7 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is)
{
if (type_ == tokenType::COMPOUND)
{
if (compoundTokenPtr_->empty())
if (data_.compoundPtr->empty())
{
FatalIOErrorInFunction(is)
<< "compound has already been transfered from token\n "
......@@ -102,16 +102,14 @@ Foam::token::compound& Foam::token::transferCompoundToken(const Istream& is)
}
else
{
compoundTokenPtr_->empty() = true;
data_.compoundPtr->empty() = true;
}
return *compoundTokenPtr_;
}
else
{
parseError("compound");
return *compoundTokenPtr_;
return *data_.compoundPtr;
}
parseError("compound");
return *data_.compoundPtr;
}
......
......@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
......@@ -25,7 +25,7 @@ Class
Foam::token
Description
A token holds items read from Istream.
A token holds an item read from Istream.
SourceFiles
tokenI.H
......@@ -64,34 +64,40 @@ Ostream& operator<<(Ostream& os, const token& t);
/*---------------------------------------------------------------------------*\
Class token Declaration
Class token Declaration
\*---------------------------------------------------------------------------*/
class token
{
public:
//- Enumeration defining the types of token
//- Enumeration defining the types of token.
// Since these values are also used to tag content in Pstream,
// the maximum number of types is limited to 30.
enum tokenType
{
UNDEFINED,
PUNCTUATION,
WORD,
VARIABLE,
STRING,
VERBATIMSTRING,
LABEL,
FLOAT_SCALAR,
DOUBLE_SCALAR,
COMPOUND,
ERROR
UNDEFINED, //!< An undefined token-type
// Fundamental types
PUNCTUATION, //!< single character punctuation
LABEL, //!< label (integer) type
FLOAT_SCALAR, //!< float (single-precision) type
DOUBLE_SCALAR, //!< double (double-precision) type
// Pointer types
WORD, //!< Contents represent a Foam::word
STRING, //!< Contents represent a Foam::string
VARIABLE, //!< Contents are a Foam::string representing a
//!< dictionary \c $variable expansion
VERBATIMSTRING, //!< Contents are a Foam::string representing verbatim
//!< content
COMPOUND, //!< Compound type such as List\<label\> etc.
ERROR //!< A token error encountered
};
//- Standard punctuation tokens
//- Standard punctuation tokens (a character)
enum punctuationToken
{
NULL_TOKEN = '\0',
......@@ -99,25 +105,26 @@ public:
TAB = '\t',
NL = '\n',
END_STATEMENT = ';',
BEGIN_LIST = '(',
END_LIST = ')',
BEGIN_SQR = '[',
END_SQR = ']',
BEGIN_BLOCK = '{',
END_BLOCK = '}',
COLON = ':',
COMMA = ',',
END_STATEMENT = ';', //!< End entry [#isseparator]
BEGIN_LIST = '(', //!< Begin list [#isseparator]
END_LIST = ')', //!< End list [#isseparator]
BEGIN_SQR = '[', //!< Begin dimensions [#isseparator]
END_SQR = ']', //!< End dimensions [#isseparator]
BEGIN_BLOCK = '{', //!< Begin block [#isseparator]
END_BLOCK = '}', //!< End block [#isseparator]
COLON = ':', //!< Colon [#isseparator]
COMMA = ',', //!< Comma [#isseparator]
HASH = '#',
ATSYM = '@',
BEGIN_STRING = '"',
END_STRING = BEGIN_STRING,
ASSIGN = '=',
ADD = '+',
SUBTRACT = '-',
MULTIPLY = '*',
DIVIDE = '/'
ASSIGN = '=', //!< Assigment/equals [#isseparator]
ADD = '+', //!< Addition [#isseparator]
SUBTRACT = '-', //!< Substract or start of negative number
MULTIPLY = '*', //!< Multiply [#isseparator]
DIVIDE = '/' //!< Divide [#isseparator]
};
......@@ -145,7 +152,6 @@ public:
//- Runtime type information
TypeName("compound");
//- Declare run-time constructor selection table
declareRunTimeSelectionTable
(
......@@ -169,7 +175,7 @@ public:
// Selectors
//- Select null constructed
static autoPtr<compound> New(const word& type, Istream&);
static autoPtr<compound> New(const word& type, Istream& is);
//- Destructor
......@@ -178,26 +184,22 @@ public:
// Member Functions
// Access
//- Return true if name is a compound type
static bool isCompound(const word& name);
//- Return true if name is a known compound type
static bool isCompound(const word& name);
bool empty() const
{
return empty_;
}
bool& empty()
{
return empty_;
}
bool empty() const
{
return empty_;
}
virtual label size() const = 0;
bool& empty()
{
return empty_;
}
// Write
virtual label size() const = 0;
virtual void write(Ostream& os) const = 0;
virtual void write(Ostream& os) const = 0;
// IOstream Operators
......@@ -236,35 +238,49 @@ public:
//- Static undefined token
static token undefinedToken;
static const token undefinedToken;
private:
//- A %union of token types
union content
{
// Fundamental values. Largest first for any {} initialization.
int64_t int64Val;
int32_t int32Val;
punctuationToken punctuationVal;
label labelVal;
floatScalar floatVal;
doubleScalar doubleVal;
// Pointers
word* wordPtr;
string* stringPtr;
mutable compound* compoundPtr;
};
// Private data