From 6b77d86563147cea3bb2e1528228dddb41b16aa0 Mon Sep 17 00:00:00 2001
From: Henry Weller <http://cfd.direct>
Date: Sun, 3 Jul 2016 22:55:16 +0100
Subject: [PATCH] wmakeCollect: Experimental scheduler for wmake to speed-up
parallel compilations
wmakeCollect collects the compilation commands for the all of the object
files to be compiled into a single makefile which is passed to make to
schedule the compilations in parallel efficiently.
Before wmakeCollect can be called the lnInclude directories must be
up-to-date and after wmakeCollect the linkage stage of the compilation
must executed using wmake.
This entire process is now handled by wmake using the new '-queue' or
'-q' option to compile sections of the OpenFOAM source tree or the
entire tree efficiently. The number of cores the compilation executes
on may be specified either using the WM_NCOMPPROCS variable or the '-j'
option.
To efficiently compile OpenFOAM after a 'git pull' the '-update' option
is provided which updates lnInclude directories, dep files and removes
deprecated files and directories. This option may be used with '-q':
wmake -q -update
---
wmake/wmake | 112 +++++++++++++++++++++++++------
wmake/wmakeCollect | 160 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 251 insertions(+), 21 deletions(-)
create mode 100755 wmake/wmakeCollect
diff --git a/wmake/wmake b/wmake/wmake
index 6d3f116d99c..0e4f6504a83 100755
--- a/wmake/wmake
+++ b/wmake/wmake
@@ -49,7 +49,7 @@
# wdep -h
#
# See also
-# wmakeLnInclude, wmakeLnIncludeAll, wdep, wrmdep, wrmo,
+# wmakeLnInclude, wmakeLnIncludeAll, wmakeQueue, wdep, wrmdep, wrmo,
# wclean, wcleanPlatform, wcleanLnIncludeAll
#
#------------------------------------------------------------------------------
@@ -64,8 +64,15 @@ Usage: $Script [OPTION] [dir]
$Script [OPTION] target [dir [MakeDir]]
options:
- -s | -silent Run make in 'silent' mode (do not echo commands)
- -a | -all Run wmake in all subdirectories
+ -s | -silent 'silent' mode (does not echo commands)
+ -a | -all wmake all sub-directories, running Allwmake if present
+ -q | -queue wmakeQueue all sub-directories, running Allwmake if present
+ -k or -non-stop Compile without stopping when errors occur
+ -j Compile using all local cores/hyperthreads
+ -jN or -j N Compile using N cores/hyperthreads
+ -no-scheduler Compile without wmakeScheduler
+ -update Update lnInclude directories, dep files, remove deprecated
+ files and directories
-h | -help Print the usage
@@ -76,7 +83,8 @@ The 'target' is a Makefile target:
e.g., platforms/linux64GccDPOpt/.../fvMesh.o
or a special target:
- all wmake all subdirectories, running Allwmake files if present
+ all wmake all sub-directories, running Allwmake if present
+ queue wmakeQueue all sub-directories, running Allwmake if present
exe Compile statically linked executable
lib Compile statically linked archive lib (.a)
libo Compile statically linked lib (.o)
@@ -104,24 +112,65 @@ all=
while [ "$#" -gt 0 ]
do
case "$1" in
- -h | -help)
- usage
- ;;
- -s | -silent)
- make="$make -s"
- shift
- ;;
- -a | -all | all)
- all="all"
- shift
- ;;
- -*)
- usage "unknown option: '$*'"
- ;;
- *)
- break
- ;;
+ # Print help
+ -h | -help)
+ usage
+ ;;
+ -s | -silent)
+ make="$make -s"
+ ;;
+ -a | -all | all)
+ all="all"
+ ;;
+ -q | -queue | queue)
+ all="queue"
+ ;;
+ # Parallel compilation on all cores of local machine
+ -j)
+ setWM_NCOMPPROCS
+ test $# -ge 2 && expr $2 + 1 > /dev/null 2>&1 \
+ && shift && export WM_NCOMPPROCS=$1
+ echo "Compiling enabled on $WM_NCOMPPROCS cores"
+ ;;
+ # Parallel compilation on specified number of cores
+ -j*)
+ export WM_NCOMPPROCS=${1#-j}
+ echo "Compiling enabled on $WM_NCOMPPROCS cores"
+ ;;
+ # Non-stop compilation, ignoring errors
+ -k | -non-stop)
+ export WM_CONTINUE_ON_ERROR=1
+ ;;
+ # Disable scheduled parallel compilation
+ -no-scheduler)
+ unset WM_SCHEDULER
+ ;;
+ # Meant to be used following a pull, this will:
+ # - remove dep files that depend on deleted files;
+ # - remove stale dep files;
+ # - update lnInclude directories;
+ # - remove empty directories, along with deprecated object directories
+ # and respective binaries.
+ -update)
+ wrmdep -update
+ wrmdep -old
+ wmakeLnIncludeAll
+ wclean empty
+ export WM_UPDATE_DEPENDENCIES=yes
+ [ -z "$all" ] && all="all"
+ ;;
+ --)
+ shift
+ break
+ ;;
+ -*)
+ usage "unknown option: '$*'"
+ ;;
+ *)
+ break
+ ;;
esac
+ shift
done
@@ -252,6 +301,27 @@ then
fi
+#------------------------------------------------------------------------------
+# Recurse the source tree to compile "all" targets using wmakeQueue
+#------------------------------------------------------------------------------
+
+#scheduler="wmakeQueue"
+scheduler="wmakeCollect"
+
+if [ "$all" = "queue" ]
+then
+ ( \
+ wmakeLnIncludeAll -j$WM_NCOMPPROCS \
+ && WM_ID=$(stat --format=%d.%i $PWD) \
+ WM_SCHEDULER=$scheduler \
+ trap '$scheduler -kill' TERM INT; \
+ wmake -all objects \
+ && $scheduler \
+ ) && wmake -all
+ exit $?
+fi
+
+
#------------------------------------------------------------------------------
# Search up the directory tree for the Make sub-directory,
# check the existance of the 'files' file and build there if present
diff --git a/wmake/wmakeCollect b/wmake/wmakeCollect
new file mode 100755
index 00000000000..4a3525a3422
--- /dev/null
+++ b/wmake/wmakeCollect
@@ -0,0 +1,160 @@
+#!/bin/bash
+#------------------------------------------------------------------------------
+# ========= |
+# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+# \\ / O peration |
+# \\ / A nd | Copyright (C) 2016 OpenFOAM Foundation
+# \\/ M anipulation |
+#-------------------------------------------------------------------------------
+# License
+# This file is part of OpenFOAM.
+#
+# OpenFOAM is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
+#
+# Script
+# wmakeCollect
+#
+# Usage
+# wmakeCollect [OPTION] <command>
+#
+# Description
+# wmake scheduler for efficient parallel compilations.
+#
+#-------------------------------------------------------------------------------
+Script=${0##*/}
+
+usage() {
+ exec 1>&2
+ while [ "$#" -ge 1 ]; do echo "$1"; shift; done
+ cat<<USAGE
+
+Usage: $Script [OPTION] <command>
+
+options:
+ -kill Removes temporary makefiles
+ -h | -help Print the usage
+
+Collecting scheduler for fast parallel compilation of large numbers of object
+files. Can be used in compiling OpenFOAM by setting the WM_SCHEDULER variable.
+
+When called with a compilation command it is written into a file in the
+directory ~.\$WM_PROJECT/.collect/$WM_ID.
+
+When called without a command the files in the ~.\$WM_PROJECT/.collect/$WM_ID
+are combined into a single Makefile which is passed to make to compile all of
+the object files in parallel.
+
+Typical usage for compiling OpenFOAM:
+
+ - Ensure all lnInclude directories are up-to-date:
+ wmakeLnIncludeAll
+
+ - Compile all with this scheduler:
+ wmake -queue or wmake -q
+
+USAGE
+ exit 1
+}
+
+# Set true to clean-up file if interupted
+cleanup=
+
+while [ "$#" -gt 0 ]
+do
+ case "$1" in
+ -h | -help)
+ usage
+ ;;
+ -kill)
+ cleanup="true"
+ shift
+ ;;
+ -*)
+ usage "unknown option: '$*'"
+ break
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+
+if [ -z "$WM_ID" ]
+then
+ echo "$Script error: build ID variable WM_ID not set"
+ exit 1
+fi
+
+# Location of the wmakeCollect makefiles
+wmakeCollectDir=$HOME/.$WM_PROJECT/.collect
+
+# Make sure directories exist
+mkdir -p $wmakeCollectDir
+
+# Location of the wmakeCollect makefiles for this build
+collectDir=$wmakeCollectDir/$WM_ID
+
+# Collected makefile for this build
+makefile="$wmakeCollectDir/Makefile.$WM_ID"
+
+
+# Clean-up files and exit
+if [ -n "$cleanup" ]
+then
+ rm -rf $collectDir
+ rm -f $makefile
+ exit 0
+fi
+
+
+if [ "$#" -gt 0 ]
+then
+ # Make sure directories exist
+ mkdir -p $collectDir
+
+ # Unique file name for makefile for the current target
+ file="$collectDir/$$_${RANDOM}"
+
+ # The current target
+ object="${@: -1:1}"
+
+ # Add the current target to the list of objects
+ echo "OBJECTS += $object" >> $file
+
+ # Add the build rule for the current target
+ echo "$object: $makefile" >> $file
+ echo -e "\t cd $PWD && \\" >> $file
+ echo -e "\t${@:1:($#-1)} $object" >> $file
+ echo >> $file
+else
+ if [ -d $collectDir ]
+ then
+ # Collect all the makefiles into a single makefiles for this build
+ cat $collectDir/* > $makefile
+
+ # Add a build rule for all of the targets
+ echo 'all: $(OBJECTS)' >> $makefile
+
+ # Clear out all of the target makefiles
+ rm -rf $collectDir
+
+ # Run make on the collected makefile
+ make -j $WM_NCOMPPROCS -f $makefile all
+
+ rm -f $makefile
+ fi
+fi
+
+#------------------------------------------------------------------------------
--
GitLab