Commit de72a04a authored by Mark Olesen's avatar Mark Olesen
Browse files

ENH: make it easier to switch between the various make dependencies programs.

- However, the new ragel-based parser is much faster
  than the others, and does not cause 'too many open files' error
  that the flex-based parser does (issue #784).

  The timings (using src/sampling as being somewhat representative)

    $ wclean; wmakeLnInclude -u .; time wmake -s dep

        3.4s  wmkdepend (ragel) [now default]
        5.7s  wmkdep (flex)
        6.1s  cpp -M

- The makeDepend script is for testing purposes only, but could used as
  a hook for other dependency generation systems (eg, ninja).
  It simply wraps 'cpp -M' in a form that is calling compatible with
  wmkdepend.

BUG: wmkdepend parser was missing optional leading space on #include match

STYLE: use -G2 (goto-based) option for wmkdepend state machine

- the machine is compact with few states and lends itself to this
parent 5f88e427
#----------------------------*- makefile-gmake -*------------------------------
# The dependency generation program
WMKDEP := $(WMAKE_BIN)/wmkdepend # 1. Ragel-based
# WMKDEP := $(WMAKE_BIN)/wmkdep # 2. Flex-based (slower, ulimit problem)
# WMKDEP := $(WM_SCRIPTS)/makeDepend # 3. cpp -M (for testing only)
WMKDEP_FLAGS := -eWM_PROJECT_DIR -eWM_THIRD_PARTY_DIR
ifneq ("$(WM_QUIET)","")
......@@ -31,7 +37,6 @@ $(OBJECTS_DIR)/%.dep : %
$(call QUIET_MESSAGE,dep,$(<F))
$(call VERBOSE_MESSAGE,Making dependency list for source file,$(<F))
@$(WM_SCRIPTS)/makeTargetDir $@
@$(WMAKE_BIN)/wmkdepend $(WMKDEP_FLAGS) -o$@ -I$(*D) $(LIB_HEADER_DIRS) $<
#flex @$(WMAKE_BIN)/wmkdep $(WMKDEP_FLAGS) -o$@ -I$(*D) $(LIB_HEADER_DIRS) $<
@$(WMKDEP) $(WMKDEP_FLAGS) -o$@ -I$(*D) $(LIB_HEADER_DIRS) $<
#------------------------------------------------------------------------------
......@@ -6,20 +6,8 @@
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
# This file is part of OpenFOAM, licensed under GNU General Public License
# <http://www.gnu.org/licenses/>.
#
# File
# wmake/scripts/AllwmakeParseArguments
......
......@@ -6,20 +6,8 @@
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
# This file is part of OpenFOAM, licensed under GNU General Public License
# <http://www.gnu.org/licenses/>.
#
# Script
# cmakeFunctions
......
#!/bin/sh
#------------------------------------------------------------------------------
# ========= |
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
# \\ / O peration |
# \\ / A nd | Copyright (C) 2018 OpenCFD Ltd.
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM, licensed under GNU General Public License
# <http://www.gnu.org/licenses/>.
#
# Script
# makeDepend
#
# Description
# Wrapping for cpp -M with argument/options compatible with
# <wmake/rules/General/transform> calls of wmkdepend or wmkdep.
#
# This is for testing purposes only, but could used as a hook for
# other dependency generation systems (eg, ninja).
#
# Note
# The order of the calling options is fragile.
# Handling of '-eENV' not working.
#
# Usage
# makeDepend [-eENV...] [-q] [-oFile] ...
#
# -eENV Environment variable path substitutions.
# -oFile Write output to File.
# -q Suppress 'No such file' warnings.
#
#------------------------------------------------------------------------------
unset envChanges output
while [ "$#" -gt 1 ]
do
case "$1" in
-q*) # quiet - ignore
;;
-e*) # -eENV - ignore for now
envChanges="$envChanges ${1#-e}"
;;
-o*) # -oFILE
output="${1#-o}"
;;
*)
break
;;
esac
shift
done
cpp -x c++ -std=c++11 -M \
-DWM_"$WM_PRECISION_OPTION" -DWM_LABEL_SIZE="$WM_LABEL_SIZE" \
$@ $output
#------------------------------------------------------------------------------
......@@ -7,20 +7,8 @@
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
# This file is part of OpenFOAM, licensed under GNU General Public License
# <http://www.gnu.org/licenses/>.
#
# Script
# makeFiles
......
......@@ -7,20 +7,8 @@
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
# This file is part of OpenFOAM, licensed under GNU General Public License
# <http://www.gnu.org/licenses/>.
#
# Script
# makeOptions
......
......@@ -7,20 +7,8 @@
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
# This file is part of OpenFOAM, licensed under GNU General Public License
# <http://www.gnu.org/licenses/>.
#
# Script
# makeTargetDir
......
......@@ -7,20 +7,8 @@
# \\/ M anipulation |
#-------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
# This file is part of OpenFOAM, licensed under GNU General Public License
# <http://www.gnu.org/licenses/>.
#
# Script
# wcleanBuild
......
......@@ -6,20 +6,8 @@
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
# This file is part of OpenFOAM, licensed under GNU General Public License
# <http://www.gnu.org/licenses/>.
#
# Script
# wmakeFunctions
......
......@@ -86,7 +86,7 @@ $(WMAKE_BIN)/wmkdepend: wmkdepend.cpp
# $(WMAKE_BIN)/wmkdepend: wmkdepend.rl
# @mkdir -p $(WMAKE_BIN)
# $(call QUIET_MESSAGE,ragel,$(<F))
# $E ragel -o $@.cpp $(<F) && $(CC) $(c++FLAGS) $(c++LESSWARN) $@.cpp -o $@
# $E ragel -G2 -o $@.cpp $(<F) && $(CC) $(c++FLAGS) $(c++LESSWARN) $@.cpp -o $@
# @rm -f $@.cpp 2>/dev/null
#------------------------------------------------------------------------------
This diff is collapsed.
......@@ -25,24 +25,32 @@ Application
wmkdepend
Description
A fast dependency list generator that emulates the behaviour and output
of cpp -M. However, the output contains no duplicates and is thus
approx. 40% faster than cpp.
It also handles missing files somewhat more gracefully.
A fast dependency list generator that emulates the behaviour and
output of cpp -M.
The algorithm uses a Ragel-generated lexer to scan for includes and
searches the files found.
The files are only visited once (their names are hashed),
which helps make this faster than cpp.
The lexer for include statements uses a Ragel-generated lexer and
then searches the files found. Each file is visited only once,
which helps make this faster than cpp. It also handles missing
files somewhat more gracefully.
The goto-based finite state machine (FSM) is generated with
ragel -G2 -o wmkdepend.cpp wmkdepend.rl
Usage
wmkdepend [-Idir..] [-iheader...] [-eENV...] [-oFile] [-q] filename
Note
May not capture all possible corner cases or line continuations such as
#include \
"file.H"
\*---------------------------------------------------------------------------*/
/*
* With cpp:
*
* cpp -x c++ -std=c++11 -nostdinc -nostdinc++
* cpp -x c++ -std=c++11
* -M -DWM_$(WM_PRECISION_OPTION) -DWM_LABEL_SIZE=$(WM_LABEL_SIZE) |
* sed -e 's,^$(WM_PROJECT_DIR)/,$$(WM_PROJECT_DIR)/,' \
* -e 's,^$(WM_THIRD_PARTY_DIR)/,$$(WM_THIRD_PARTY_DIR)/,'
......@@ -57,7 +65,7 @@ Usage
#include <unordered_set>
#include <vector>
// Sizing for read buffer
// Length of the input read buffer
#define READ_BUFLEN 16384
// The executable name (for messages), without requiring access to argv[]
......@@ -78,7 +86,6 @@ void usage()
" -eENV Environment variable path substitutions.\n"
" -oFile Write output to File.\n"
" -q Suppress 'No such file' warnings.\n"
" -v Some verbosity.\n"
"\nDependency list generator, similar to 'cpp -M'\n\n";
}
......@@ -160,7 +167,7 @@ namespace Files
// /openfoam/project/path/directory/xyz
// -> $(WM_PROJECT_DIR)/directory/xyz
//
FILE* fopen_file(std::string fileName)
FILE* fopen_file(const std::string& fileName)
{
const auto len = fileName.size();
const char *fname = fileName.c_str();
......@@ -262,7 +269,7 @@ namespace Files
std::cerr << ": " << strerror(errno);
}
std::cerr << "\n" << std::flush;
std::cerr << '\n' << std::flush;
}
return filePtr;
......@@ -273,37 +280,34 @@ namespace Files
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
// Ragel machine requirements: text start/end, action, code state
// Ragel machine requirements: token start/end, action, code state
// are defined later (prior to use)
%%{
machine scanInclude;
machine wmkdep;
write data nofinal;
action start_incl { include_start = fpc; }
hashInclude = space* '#' space* 'include' space* ; ## include directive
includeName = [^\"<>]+ ; ## Text between "" or <>
action begInclude { ts_inclName = fpc; } ## Remember start position
action end_incl
## No 'if (ts_inclName) ...' guard needed (implicit via the FSM)
action endInclude
{
if (include_start)
{
processFile(std::string(include_start, (fpc - include_start)));
}
include_start = nullptr;
// std::cerr << std::string(ts, (te-ts)) << '\n';
processFile(std::string(ts_inclName, (fpc - ts_inclName)));
}
consume_comment :=
any* :>> '*/' @{ fgoto main; };
user_include =
'#' space* 'include' space* '"' %start_incl [^\"]+ %end_incl '"' ;
consume_comment := any* :>> '*/' @{ fgoto main; };
main := |*
# Single and double strings
# Single and double quoted strings
( 'L'? "'" ( [^'\\\n] | /\\./ )* "'") ; # " swallow
( 'L'? '"' ( [^"\\\n] | /\\./ )* '"') ; # ' swallow
user_include ;
hashInclude '"' %begInclude includeName %endInclude '"' ;
'/*' { fgoto consume_comment; };
'//' [^\n]* '\n' ;
......@@ -323,67 +327,72 @@ void processFile(const std::string& fileName)
FILE* infile = Files::open(fileName);
if (!infile) return;
// Ragel text start/end, action, code state
char *ts = nullptr, *te = nullptr;
// ts, te = Ragel token start/end points (required naming)
// act, cs = Ragel action, code state, respectively (required naming)
char *ts, *te;
unsigned act, cs;
// For the include action:
char *include_start = nullptr;
%% write init ; ## /* FSM initialization here */;
%%{ write init; }%%
// Token start of include filename (for begInclude, endInclude actions)
char *ts_inclName = nullptr;
// Buffering
char buf[READ_BUFLEN];
size_t bytesPending = 0;
char inbuf[READ_BUFLEN];
size_t pending = 0;
// Do the first read
// Processing loop (as per Ragel pdf example)
for (bool good = true; good; /*nil*/)
{
const size_t avail = READ_BUFLEN - bytesPending;
const size_t avail = READ_BUFLEN - pending;
if (!avail)
{
// We overfilled the buffer while trying to scan a token...
std::cerr
<< "OUT OF BUFFER SPACE while scanning " << fileName << '\n';
<< EXENAME ": buffer full while scanning '"
<< fileName << "'\n";
break;
}
char *p = buf + bytesPending;
const size_t bytesRead = ::fread(p, 1, avail, infile);
// p, pe = Ragel parsing point and parsing end (required naming)
// eof = Ragel EOF point (required naming)
char *pe = p + bytesRead;
char *p = inbuf + pending;
const size_t gcount = ::fread(p, 1, avail, infile);
char *pe = p + gcount;
char *eof = nullptr;
// If we see eof then append the EOF char.
if (feof(infile))
if (!gcount) // Could also use feof(infile)
{
// Tag 'pe' as being the EOF for the FSM as well
eof = pe;
good = false;
}
%%{ write exec; }%%
%% write exec ; ## /* FSM execution here */;
if (cs == scanInclude_error)
if (cs == wmkdep_error)
{
// Machine failed before finding a token
std::cerr << "PARSE ERROR while scanning " << fileName << '\n';
// FSM failed before finding a token
std::cerr
<< EXENAME ": parse error while scanning '"
<< fileName << "'\n";
break;
}
// Now set up the prefix.
if (ts == nullptr)
if (ts)
{
bytesPending = 0;
// Preserve incomplete token
pending = pe - ts;
::memmove(inbuf, ts, pending);
te = inbuf + (te - ts); // token end (after memmove)
ts = inbuf; // token start
}
else
{
// There are data that needs to be shifted over.
bytesPending = pe - ts;
::memmove(buf, ts, bytesPending);
te -= (ts-buf);
ts = buf;
pending = 0;
}
}
fclose(infile);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment