Commit 49a70d83 authored by Mark Olesen's avatar Mark Olesen
Browse files

ENH: add C++-based wmkdepend parser (uses Coco/R grammar).

- This avoids dependency on lex/flex and provides better encapsulation
  for buffer switching. As a result, the maximum number of open files
  only corresponds to the include depth.
parent 00616b72
......@@ -2,7 +2,7 @@
# ========= |
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
# \\ / O peration |
# \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd.
# \\ / A nd | Copyright (C) 1991-2010 OpenCFD Ltd.
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
......@@ -60,16 +60,17 @@ include $(RULES)/$(WM_LINK_LANGUAGE)
# targets
#------------------------------------------------------------------------------
all: $(BIN)/dirToString $(BIN)/wmkdep
all: $(BIN)/dirToString $(BIN)/wmkdep $(BIN)/wmkdepend
clean:
rm -f $(BIN)/dirToString $(BIN)/wmkdep 2>/dev/null
rm -f $(BIN)/dirToString $(BIN)/wmkdep $(BIN)/wmkdepend 2>/dev/null
$(BIN)/dirToString: dirToString.c
@mkdir -p $(BIN)
$(cc) $(cFLAGS) dirToString.c -o $(BIN)/dirToString
$(BIN)/wmkdep: wmkdep.l
@mkdir -p $(BIN)
flex wmkdep.l
......@@ -77,4 +78,13 @@ $(BIN)/wmkdep: wmkdep.l
@rm -f lex.yy.c 2>/dev/null
# for bootstrapping - use generated files directly (instead of from .atg file)
$(BIN)/wmkdepend: wmkdepend.cpp \
wmkdependParser.cpp wmkdependScanner.cpp \
wmkdependParser.h wmkdependScanner.h
@mkdir -p $(BIN)
$(CC) $(c++FLAGS) \
wmkdepend.cpp wmkdependParser.cpp wmkdependScanner.cpp \
-o $(BIN)/wmkdepend
#------------------------------------------------------------------------------
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2010-2010 OpenCFD Ltd.
\\/ M anipulation |
------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Application
wmkdepend
Description
A fast dependency list generator that emulates the behaviour and
output of cpp -M. However, the output contains no duplications and
is ~40% faster than cpp.
The algorithm uses flex to scan for includes and searches the files
found. Each file is entered into a hash table so that files are scanned
only once. This is why this program is faster than cpp.
Usage
wmkdep [ -Idirectory ... -Idirectory ] filename
\*---------------------------------------------------------------------------*/
#include <cstdio>
#include <stdlib.h>
#include <cstring>
#include "wmkdependParser.h"
// Note: since we use the Coco/R default error messages, we must use
// wide streams for stderr.
void printUsage(const char* message = NULL)
{
if (message)
{
fwprintf(stderr, L"\nError: %s\n\n", message);
}
fwprintf
(
stderr,
L"Usage: wmkdepend [ -Idirectory ... -Idirectory ] filename\n"
);
}
int main(int argc, char* argv[])
{
if (argc == 1)
{
printUsage("Error: input file not supplied");
::exit(1);
}
for (int i=1; i < argc; i++)
{
if (strncmp(argv[i], "-I", 2) == 0 && strlen(argv[i]) > 2)
{
std::string dirName(argv[i] + 2);
// add trailing slash if required
if (dirName.rfind('/') != dirName.size()-1)
{
dirName += '/';
}
wmake::Parser::includeDirs.push_back(dirName);
}
}
std::string sourceFile(argv[argc-1]);
fwprintf
(
stderr,
L"Making dependency list for source file %s\n",
sourceFile.c_str()
);
std::string::size_type basePos = sourceFile.rfind('/');
if (basePos == std::string::npos)
{
basePos = 0;
}
else
{
basePos++;
}
std::string::size_type dotPos = sourceFile.rfind('.');
if
(
dotPos == std::string::npos
|| dotPos == sourceFile.size()-1
|| dotPos <= basePos
)
{
fwprintf
(
stderr,
L"Cannot find extension in source file name %s\n",
sourceFile.c_str()
);
::exit(1);
}
std::string depFile = sourceFile.substr(0, dotPos);
depFile += ".dep";
const std::string sourceExt = sourceFile.substr(dotPos);
if (sourceExt == ".java")
{
// import directories to ignore
wmake::Parser::ignoreDir("java.*");
wmake::Parser::ignoreDir("org.*");
wmake::Parser::ignoreDir("com.*");
wmake::Parser::ignoreDir("sunw.*");
wmake::Parser::ignoreDir("sun.*");
wmake::Parser::ignoreDir("launcher.*");
std::cout
<< "$(CLASSES_DIR)/"
<< sourceFile.substr(basePos, dotPos - basePos) << ".class: "
<< depFile << "\n";
}
else
{
std::cout
<< "$(OBJECTS_DIR)/"
<< sourceFile.substr(basePos, dotPos - basePos) << ".o: "
<< depFile << "\n";
}
wmake::Parser::sourceFile = sourceFile;
wmake::Parser::depFile = depFile;
wmake::Parser::includeFile(sourceFile);
return 0;
}
/*****************************************************************************/
/*---------------------------------------------------------------------------*\
Attributed Grammar for Coco/R (-*- C++ -*- version)
compile with:
coco-cpp wmkdependParser.atg
\*---------------------------------------------------------------------------*/
[copy]
/*---------------------------------*- C++ -*---------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2010-2010 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
@file wmkdependParser.atg
Description
An attributed Coco/R grammar to parse C/C++, Fortran and Java files
for include and import statements.
SourceFiles
generated
\*---------------------------------------------------------------------------*/
[/copy]
#include <iostream>
#include <string>
#include <list>
//! @brief A simple HashTable implementation
/**
* @note This hash table is only vaguely STL-like. In accordance with
* its present purpose, this hash table only supports a constIterator
* and no deletions. For simplicity, the constIterator increment is
* simply via a next() method. Instead of comparing to an end value,
* the constIterator valid() method is used.
* For example,
* @code
* for
* (
* HashTable<foo>::constIterator iter = myHash.begin();
* iter.valid();
* iter.next()
* )
* {
* std::cerr<< "key: " << iter.key() << "\n";
* }
* @endcode
*
*/
class StringHashSet
{
//! An entry within the HashTable
struct hashedEntry
{
const std::string key_; //<! The lookup key
hashedEntry *next_; //<! Pointer to next hashedEntry in sub-list
hashedEntry(const std::string& key, hashedEntry *next=0)
:
key_(key), next_(next)
{}
};
const int size_; //<! fixed HashTable size
hashedEntry** table_;
public:
//! Construct with a default size
StringHashSet(int size = 500)
:
size_(size),
table_(new hashedEntry*[size_])
{
memset(table_, 0, size_ * sizeof(hashedEntry*));
}
//! Destructor
~StringHashSet()
{
for (int hashIdx = 0; hashIdx < size_; ++hashIdx)
{
hashedEntry* ep = table_[hashIdx];
while (ep)
{
hashedEntry* del = ep;
ep = ep->next_;
delete del;
}
}
delete[] table_;
table_ = 0;
}
//! Return hash index for lookup name in hash table
bool hashKeyIndex(const std::string& name) const
{
int hashIdx = 0;
// calculate hash index
for
(
std::string::const_iterator iter = name.begin();
iter != name.end();
++iter
)
{
hashIdx = hashIdx << 1 ^ *iter;
}
if (hashIdx < 0)
{
hashIdx = -hashIdx;
}
return hashIdx % size_;
}
//! Return true if name is found in hash table
bool found(const std::string& name) const
{
const int hashIdx = hashKeyIndex(name);
for (hashedEntry* ep = table_[hashIdx]; ep; ep = ep->next_)
{
if (name == ep->key_)
{
// found
return true;
}
}
// entry not found
return false;
}
//! Return true if name is found in hash table, insert if not found
bool foundOrInsert(const std::string& name)
{
const int hashIdx = hashKeyIndex(name);
for (hashedEntry* ep = table_[hashIdx]; ep; ep = ep->next_)
{
if (name == ep->key_)
{
// found - return true
return true;
}
}
// not found - insert it
table_[hashIdx] = new hashedEntry(name, table_[hashIdx]);
// entry not found (but was added) - return false
return false;
}
};
/*---------------------------------------------------------------------------*/
COMPILER wmkdepend
// grammar pragmas:
$namespace=wmake
$prefix=wmkdepend
$define=FORCE_UTF8
/*---------------------------------------------------------------------------*/
private:
//! Hash of files already visited
static StringHashSet visitedFiles_;
//! Hash of (java) directories already visited
static StringHashSet visitedDirs_;
//! Replace all '.' with '/'
static void dotToSlash(std::string& name);
//! Import (java) directories
static void importDir(const std::string& dirName);
//! Import (java) file
static void importFile(const std::string& name);
public:
//! Include directories to search
static std::list<std::string> includeDirs;
//! The name of the top-level source file
static std::string sourceFile;
//! The name of the top-level dep file
static std::string depFile;
//! Add directory to list of visited dirs, thus effectively ignoring it
static void ignoreDir(const std::string& name);
//! Include file
static void includeFile(const std::string& name);
/*---------------------------------------------------------------------------*/
[code]
#include <sys/types.h>
#include <dirent.h>
StringHashSet Parser::visitedFiles_;
StringHashSet Parser::visitedDirs_;
std::list<std::string> Parser::includeDirs;
std::string Parser::sourceFile;
std::string Parser::depFile;
void Parser::dotToSlash(std::string& name)
{
std::string::size_type start = 0;
while ((start = name.find('.', start)) != std::string::npos)
{
name.replace(start, 1, 1, '/');
start++;
}
}
void Parser::ignoreDir(const std::string& name)
{
visitedDirs_.foundOrInsert(name);
}
void Parser::includeFile(const std::string& name)
{
if (visitedFiles_.foundOrInsert(name))
{
return;
}
// use stdio and buffering within Coco/R -- (faster)
FILE *fh = fopen(name.c_str(), "r");
if (fh)
{
std::cout << depFile << ": " << name << "\n";
}
else
{
for
(
std::list<std::string>::const_iterator iter = includeDirs.begin();
iter != includeDirs.end();
++iter
)
{
const std::string pathName = *iter + name;
fh = fopen(pathName.c_str(), "r");
if (fh)
{
std::cout << depFile << ": " << pathName << "\n";
break;
}
}
}
if (fh)
{
Scanner scanner(fh);
Parser parser(&scanner);
parser.Parse();
fclose(fh);
}
else
{
fwprintf
(
stderr,
L"could not open file %s for source file %s\n",
name.c_str(), sourceFile.c_str()
);
}
}
void Parser::importFile(const std::string& name)
{
// check if a globbed form was already visited
std::string::size_type dotPos = name.find('.');
if (dotPos != std::string::npos)
{
std::string dirGlob = name.substr(0, dotPos);
dirGlob += ".*";
if (visitedDirs_.found(dirGlob))
{
return;
}
}
std::string javaFileName = name;
dotToSlash(javaFileName);
javaFileName += ".java";
includeFile(javaFileName);
}
void Parser::importDir(const std::string& name)
{
if (visitedDirs_.foundOrInsert(name))
{
return;
}
std::string dirName = name;
dotToSlash(dirName);
DIR *source = opendir(dirName.c_str());
if (source)
{
struct dirent *list;
// Read and parse all the entries in the directory
while ((list = readdir(source)) != NULL)
{
const char* ext = strstr(list->d_name, ".java");
// avoid matching on something like '.java~'
if (ext && strlen(ext) == 5)
{
std::string pathName = dirName + list->d_name;
includeFile(pathName);
}
}
closedir(source);
}
else
{
fwprintf
(
stderr,
L"could not open directory %s\n",
dirName.c_str()
);
return;
}
}
[/code]
/*---------------------------------------------------------------------------*/
CHARACTERS
letter = 'A'..'Z' + 'a'..'z' + '_'.
digit = "0123456789".
cr = '\r'.
lf = '\n'.
tab = '\t'.
stringCh = ANY - '"' - '\\' - cr - lf.
printable = '\u0020' .. '\u007e'.
java_letter = letter + '$'.
// * * * * * * * * * * * * * * * * TOKENS * * * * * * * * * * * * * * * * * //
TOKENS
// string
string =
'"' { stringCh | '\\' printable } '"'.
// single-quoted string (eg, Fortran)
sqstring =
'\'' { stringCh | '\\' printable } '\''.
// for java import
package_name =
java_letter { java_letter | digit }
{ '.' java_letter { java_letter | digit } } .
// for java import
package_dir =
java_letter { java_letter | digit }
{ '.' java_letter { java_letter | digit } } ".*" .
// * * * * * * * * * * * PRAGMAS / COMMENTS / IGNORE * * * * * * * * * * * //
COMMENTS FROM "/*" TO "*/" NESTED
COMMENTS FROM "//" TO lf
IGNORE tab
// * * * * * * * * * * * * * * * PRODUCTIONS * * * * * * * * * * * * * * * //
PRODUCTIONS
wmkdepend
=
{
// C/C++-style includes
'#'
[
"include"
[
string (.
if (isUTF8())
{
includeFile(t->toStringUTF8(1, t->length()-2));
}
else
{
includeFile(t->toString(1, t->length()-2));
}
.)
]
]
[ ANY { ANY } ] '\n' // skip trailing junk
// Fortran-style includes
| "include"