ENH: preserve tokenized directive and variable types

src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.C

@@ -227,8 +227,9 @@ Foam::Istream& Foam::UIPstream::read(token& t)
             return *this;
         }
 
-        // Word
+        // Word/directive
         case token::tokenType::WORD :
+        case token::tokenType::DIRECTIVE :
         {
             word val;
             if (read(val))
@@ -240,6 +241,7 @@ Foam::Istream& Foam::UIPstream::read(token& t)
                 else
                 {
                     t = std::move(val);
+                    t.setType(token::tokenType(c));
                 }
             }
             else

src/OpenFOAM/db/IOstreams/Pstreams/UOPstream.C

@@ -202,6 +202,14 @@ bool Foam::UOPstream::write(const token& tok)
             return true;
         }
 
+        case token::tokenType::DIRECTIVE :
+        {
+            writeToBuffer(char(token::tokenType::DIRECTIVE));
+            writeStringToBuffer(tok.wordToken());
+
+            return true;
+        }
+
         case token::tokenType::VARIABLE :
         case token::tokenType::VERBATIM :
         {

src/OpenFOAM/db/IOstreams/Sstreams/ISstream.C

@@ -253,10 +253,21 @@ Foam::Istream& Foam::ISstream::read(token& t)
             else
             {
                 // Word beginning with '#'. Eg, "#include"
+                // Put back both so that '#...' is included in the directive
+
                 putback(nextC);
                 putback(c);
 
-                readWordToken(t);
+                word val;
+                if (read(val).bad())
+                {
+                    t.setBad();
+                }
+                else
+                {
+                    t = std::move(val); // Move contents to token
+                    t.setType(token::tokenType::DIRECTIVE);
+                }
             }
 
             return *this;

src/OpenFOAM/db/IOstreams/Sstreams/ISstream.H

@@ -68,7 +68,7 @@ class ISstream
         //- Get the next valid character
         char nextValid();
 
-        //- Get a word token
+        //- Read a word token
         void readWordToken(token& t);
 
         //- Read a verbatim string (excluding block delimiters).

src/OpenFOAM/db/IOstreams/Sstreams/OSstream.C

@@ -45,8 +45,17 @@ bool Foam::OSstream::write(const token& tok)
             return true;
         }
 
+        case token::tokenType::DIRECTIVE :
+        {
+            // The '#' sigil is already part of the wordToken
+            write(tok.wordToken());
+
+            return true;
+        }
+
         case token::tokenType::VERBATIM :
         {
+            // Surrounding '#{ .. #}' to be recognized as verbatim
             write(char(token::HASH));
             write(char(token::BEGIN_BLOCK));
             writeQuoted(tok.stringToken(), false);

src/OpenFOAM/db/IOstreams/Sstreams/prefixOSstream.C

@@ -70,39 +70,7 @@ void Foam::prefixOSstream::print(Ostream& os) const
 
 bool Foam::prefixOSstream::write(const token& tok)
 {
-    // Direct token handling only for some types
-
-    switch (tok.type())
-    {
-        case token::tokenType::FLAG :
-        {
-            // silently consume the flag
-            return true;
-        }
-
-        case token::tokenType::VERBATIM :
-        {
-            write(char(token::HASH));
-            write(char(token::BEGIN_BLOCK));
-            writeQuoted(tok.stringToken(), false);
-            write(char(token::HASH));
-            write(char(token::END_BLOCK));
-
-            return true;
-        }
-
-        case token::tokenType::VARIABLE :
-        {
-            writeQuoted(tok.stringToken(), false);
-
-            return true;
-        }
-
-        default:
-            break;
-    }
-
-    return false;
+    return OSstream::write(tok);
 }
 
 

src/OpenFOAM/db/IOstreams/token/token.H

@@ -89,6 +89,7 @@ public:
         // Pointer types
         WORD,             //!< A Foam::word
         STRING,           //!< A string
+        DIRECTIVE,        //!< A dictionary \c \#directive (word variant)
         VARIABLE,         //!< A dictionary \c \$variable (string variant)
         VERBATIM,         //!< Verbatim string content
         COMPOUND,         //!< Compound type such as \c List\<label\> etc.
@@ -384,7 +385,7 @@ public:
         //  use the corresponding assignment operator.
         //
         //  \return true if the change was successful or no change was required
-        inline bool setType(const tokenType variant);
+        inline bool setType(const tokenType tokType);
 
         //- The line number for the token
         inline label lineNumber() const;
@@ -428,19 +429,22 @@ public:
         //- Token is LABEL, FLOAT or DOUBLE
         inline bool isNumber() const;
 
-        //- Token is WORD
+        //- Token is WORD or DIRECTIVE word
         inline bool isWord() const;
 
+        //- Token is DIRECTIVE (word variant)
+        inline bool isDirective() const;
+
         //- Token is STRING, VARIABLE or VERBATIM string
         inline bool isString() const;
 
-        //- Token is VARIABLE
+        //- Token is VARIABLE (string variant)
         inline bool isVariable() const;
 
-        //- Token is VERBATIM string
+        //- Token is VERBATIM string (string variant)
         inline bool isVerbatim() const;
 
-        //- Token is WORD, STRING, VARIABLE or VERBATIM
+        //- Token is WORD, DIRECTIVE, STRING, VARIABLE or VERBATIM
         inline bool isStringType() const;
 
         //- Token is COMPOUND
@@ -484,12 +488,13 @@ public:
         inline scalar number() const;
 
         //- Return const reference to the word contents.
-        //  Report FatalIOError and return \b "" if token is not a WORD
+        //  Report FatalIOError and return \b "" if token is not a
+        //  WORD or DIRECTIVE
         inline const word& wordToken() const;
 
         //- Return const reference to the string contents.
         //  Report FatalIOError and return \b "" if token is not a
-        //  STRING, VARIABLE, VERBATIM or an upcast WORD
+        //  STRING, VARIABLE, VERBATIM or an upcast WORD or DIRECTIVE
         inline const string& stringToken() const;
 
         //- Read access for compound token

src/OpenFOAM/db/IOstreams/token/tokenI.H

@@ -114,6 +114,7 @@ inline Foam::token::token(const token& tok)
     switch (type_)
     {
         case tokenType::WORD:
+        case tokenType::DIRECTIVE:
         {
             data_.wordPtr = new word(*tok.data_.wordPtr);
             break;
@@ -247,6 +248,7 @@ inline void Foam::token::reset()
     switch (type_)
     {
         case tokenType::WORD:
+        case tokenType::DIRECTIVE:
         {
             delete data_.wordPtr;
             break;
@@ -300,15 +302,15 @@ inline Foam::token::tokenType Foam::token::type() const
 }
 
 
-inline bool Foam::token::setType(token::tokenType variant)
+inline bool Foam::token::setType(token::tokenType tokType)
 {
-    if (type_ == variant)
+    if (type_ == tokType)
     {
         // No change required
         return true;
     }
 
-    switch (variant)
+    switch (tokType)
     {
         case tokenType::BOOL:
         case tokenType::LABEL:
@@ -317,7 +319,24 @@ inline bool Foam::token::setType(token::tokenType variant)
             {
                 case tokenType::BOOL:
                 case tokenType::LABEL:
-                    type_ = variant;
+                    type_ = tokType;
+                    return true;
+                    break;
+
+                default:
+                    break;
+            }
+        }
+        break;
+
+        case tokenType::WORD:
+        case tokenType::DIRECTIVE:
+        {
+            switch (type_)
+            {
+                case tokenType::WORD:
+                case tokenType::DIRECTIVE:
+                    type_ = tokType;
                     return true;
                     break;
 
@@ -337,7 +356,7 @@ inline bool Foam::token::setType(token::tokenType variant)
                 case tokenType::STRING:
                 case tokenType::VARIABLE:
                 case tokenType::VERBATIM:
-                    type_ = variant;
+                    type_ = tokType;
                     return true;
                     break;
 
@@ -553,13 +572,27 @@ inline Foam::scalar Foam::token::number() const
 
 inline bool Foam::token::isWord() const
 {
-    return (type_ == tokenType::WORD);
+    return
+    (
+        type_ == tokenType::WORD
+     || type_ == tokenType::DIRECTIVE
+    );
+}
+
+
+inline bool Foam::token::isDirective() const
+{
+    return (type_ == tokenType::DIRECTIVE);
 }
 
 
 inline const Foam::word& Foam::token::wordToken() const
 {
-    if (type_ == tokenType::WORD)
+    if
+    (
+        type_ == tokenType::WORD
+     || type_ == tokenType::DIRECTIVE
+    )
     {
         return *data_.wordPtr;
     }
@@ -609,10 +642,14 @@ inline const Foam::string& Foam::token::stringToken() const
     {
         return *data_.stringPtr;
     }
-    else if (type_ == tokenType::WORD)
+    else if
+    (
+        type_ == tokenType::WORD
+     || type_ == tokenType::DIRECTIVE
+    )
     {
-        // Upcast to string
-        return static_cast<const string&>(*data_.wordPtr);
+        // Foam::word derives from Foam::string, no need to cast.
+        return *data_.wordPtr;
     }
 
     parseError("string");
@@ -666,6 +703,7 @@ inline void Foam::token::operator=(const token& tok)
     switch (type_)
     {
         case tokenType::WORD:
+        case tokenType::DIRECTIVE:
         {
             data_.wordPtr = new word(*tok.data_.wordPtr);
         }
@@ -809,6 +847,7 @@ inline bool Foam::token::operator==(const token& tok) const
             return equal(data_.doubleVal, tok.data_.doubleVal);
 
         case tokenType::WORD:
+        case tokenType::DIRECTIVE:
             return *data_.wordPtr == *tok.data_.wordPtr;
 
         case tokenType::STRING:
@@ -837,7 +876,7 @@ inline bool Foam::token::operator==(const std::string& s) const
 {
     return
     (
-        type_ == tokenType::WORD
+        isWord()
       ? s == *data_.wordPtr
       : isString() && s == *data_.stringPtr
     );

src/OpenFOAM/db/IOstreams/token/tokenIO.C

@@ -74,6 +74,10 @@ static OS& printTokenInfo(OS& os, const token& tok)
             os  << "word '" << tok.wordToken() << '\'';
         break;
 
+        case token::tokenType::DIRECTIVE:
+            os  << "directive '" << tok.wordToken() << '\'';
+        break;
+
         case token::tokenType::STRING:
             os  << "string " << tok.stringToken();
         break;
@@ -135,6 +139,7 @@ Foam::word Foam::token::name() const
         case token::tokenType::FLOAT: return "float";
         case token::tokenType::DOUBLE: return "double";
         case token::tokenType::WORD: return "word";
+        case token::tokenType::DIRECTIVE: return "directive";
         case token::tokenType::STRING: return "string";
         case token::tokenType::VERBATIM: return "verbatim";
         case token::tokenType::VARIABLE: return "variable";
@@ -189,6 +194,13 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const token& tok)
             os << tok.data_.doubleVal;
         break;
 
+        // Different behaviour for (serial/parallel) streams: preserve types
+        case token::tokenType::DIRECTIVE:
+        case token::tokenType::VARIABLE:
+        case token::tokenType::VERBATIMSTRING:
+            os.write(tok);
+        break;
+
         case token::tokenType::WORD:
             os << *tok.data_.wordPtr;
         break;
@@ -197,13 +209,6 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const token& tok)
             os << *tok.data_.stringPtr;
         break;
 
-        case token::tokenType::VARIABLE:
-        case token::tokenType::VERBATIMSTRING:
-            // Different behaviour for (serial/parallel) stream type
-            // - preserve its type
-            os.write(tok);
-        break;
-
         case token::tokenType::COMPOUND:
             os << *tok.data_.compoundPtr;
         break;

src/OpenFOAM/db/dictionary/functionEntries/ifEntry/ifEntry.C

@@ -85,6 +85,7 @@ bool Foam::functionEntries::ifEntry::execute
     string line;
     dynamic_cast<ISstream&>(is).getLine(line);
     line += ';';
+
     IStringStream lineStream(line);
     const primitiveEntry e("ifEntry", parentDict, lineStream);
 

src/OpenFOAM/db/dictionary/functionEntries/ifeqEntry/ifeqEntry.C

@@ -155,14 +155,15 @@ bool Foam::functionEntries::ifeqEntry::equalToken
             return (eqType && t1.pToken() == t2.pToken());
 
         case token::WORD:
-            if (eqType)
+        case token::DIRECTIVE:
+            if (t2.isWord())
             {
                 return t1.wordToken() == t2.wordToken();
             }
             else if (t2.isString())
             {
-                wordRe w2(t2.stringToken(), wordRe::DETECT);
-                return w2.match(t1.wordToken(), false);
+                const wordRe w2(t2.stringToken(), wordRe::DETECT);
+                return w2.match(t1.wordToken());
             }
             return false;
 
@@ -171,25 +172,21 @@ bool Foam::functionEntries::ifeqEntry::equalToken
             {
                 const wordRe w1(t1.stringToken(), wordRe::DETECT);
                 const wordRe w2(t2.stringToken(), wordRe::DETECT);
-                return w1.match(w2, false) || w2.match(w1, false);
+                return w1.match(w2) || w2.match(w1);
             }
             else if (t2.isWord())
             {
                 const wordRe w1(t1.stringToken(), wordRe::DETECT);
-                return w1.match(t2.wordToken(), false);
+                return w1.match(t2.wordToken());
             }
             return false;
 
         case token::VARIABLE:
         case token::VERBATIM:
-            if (eqType)
+            if (t2.isStringType())
             {
                 return t1.stringToken() == t2.stringToken();
             }
-            else if (t2.isWord())
-            {
-                return t1.stringToken() == t2.wordToken();
-            }
             return false;
 
         case token::LABEL:
@@ -248,18 +245,24 @@ void Foam::functionEntries::ifeqEntry::skipUntil
     {
         token t;
         readToken(t, is);
-        if (t.isWord())
+
+        if (!t.isDirective())
         {
-            if (t.wordToken() == "#if" || t.wordToken() == "#ifeq")
-            {
-                stack.append(filePos(is.name(), is.lineNumber()));
-                skipUntil(stack, parentDict, "#endif", is);
-                stack.remove();
-            }
-            else if (t.wordToken() == endWord)
-            {
-                return;
-            }
+            continue;
+        }
+        else if
+        (
+            t.wordToken() == "#if"
+         || t.wordToken() == "#ifeq"
+        )
+        {
+            stack.append(filePos(is.name(), is.lineNumber()));
+            skipUntil(stack, parentDict, "#endif", is);
+            stack.remove();
+        }
+        else if (t.wordToken() == endWord)
+        {
+            return;
         }
     }
 
@@ -342,6 +345,7 @@ bool Foam::functionEntries::ifeqEntry::execute
         while (!is.eof())
         {
             readToken(t, is);
+
             if
             (
                 t.isWord()

src/OpenFOAM/db/dictionary/primitiveEntry/primitiveEntryIO.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2015 OpenFOAM Foundation
-    Copyright (C) 2017-2019 OpenCFD Ltd.
+    Copyright (C) 2017-2020 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -61,29 +61,28 @@ bool Foam::primitiveEntry::acceptToken
 {
     bool accept = tok.good();
 
-    if (tok.isWord())
+    if (tok.isDirective())
     {
+        // Directive: wordToken starts with '#'
         const word& key = tok.wordToken();
 
         accept =
         (
             disableFunctionEntries
          || key.size() == 1
-         || (
-                !(key[0] == '$' && expandVariable(key.substr(1), dict))
-             && !(key[0] == '#' && expandFunction(key.substr(1), dict, is))
-            )
+         || !expandFunction(key.substr(1), dict, is)
         );
     }
     else if (tok.isVariable())
     {
+        // Variable: stringToken starts with '$'
         const string& key = tok.stringToken();
 
         accept =
         (
             disableFunctionEntries
          || key.size() == 1
-         || !(key[0] == '$' && expandVariable(key.substr(1), dict))
+         || !expandVariable(key.substr(1), dict)
         );
     }
 
@@ -304,7 +303,7 @@ void Foam::primitiveEntry::write(Ostream& os) const
 }
 
 
-// * * * * * * * * * * * * * Ostream Operator  * * * * * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * IOstream Operators  * * * * * * * * * * * * //
 
 template<>
 Foam::Ostream& Foam::operator<<