Newer
Older
#!/bin/sh
#------------------------------------------------------------------------------
# ========= |
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
# \\ / O peration |
# \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
# \\/ M anipulation |
#-------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
#
# Script
# foamCheckJobs
#
# Description
# Uses runningJobs/, finishedJobs/ and foamProcessInfo to create stateFile.
# stateFile contains per pid information on state of process. Format:
# pid state command
#
# where state is one of 'RUNN', 'SUSP', 'OTHR', 'FINI', 'ABRT' ('PEND')
# (first three are from foamProcessInfo, others from jobInfo files)
# (PEND is special state from when user has submitted but no jobInfo
# file yet. Not supported by this script yet)
#
#------------------------------------------------------------------------------
Script=${0##*/}
#-------------------------------------------------------------------------------
#- User settings
#- Number of days for files to be considered old
NDAYSLIMIT=7
#-------------------------------------------------------------------------------
#- work file
TMPFILE=/tmp/${Script}$$.tmp
#- work dir. Needs to be accessible for all machines
MACHDIR=$HOME/.OpenFOAM/${Script}
DEFSTATEFILE=$HOME/.OpenFOAM/foamCheckJobs.out
# The default is "~/.OpenFOAM/jobControl"
: ${FOAM_JOB_DIR:=$HOME/.OpenFOAM/jobControl}
if [ `uname -s` = Linux ]
then
ECHO='echo -e'
ECHO='echo'
usage() {
cat<<USAGE
Usage: $Script [stateFile]
This program checks all the locks in the FOAM_JOB_DIR directory to see if
their processes are still running. Processes will not release their
lock if they exit abnormally. This program will try to obtain process
information on the machine the process ran on and release the lock
if the program is no longer running.
Note: all machines have to be reachable using ssh.
The output from checking all running jobs is collected in an optional
file.
FILES:
\$FOAM_JOB_DIR/runningJobs locks for running processes
/finishedJobs locks for finished processes
USAGE
exit 1
}
#-------------------------------------------------------------------------------
#
# Functions
#
#-------------------------------------------------------------------------------
# getRawEntry dictionary entry
# Prints value of dictionary entry
getRawEntry() {
grep -v '^//' $1 | grep "^[ \t]*$2 " | sed -e "s/^[ \t]*$2 [ ]*//"
}
# getEntry dictionary entry
# Like getRawEntry but strips " and ending ';'
getEntry() {
getRawEntry $1 $2 | sed -e 's/^"//' -e 's/;$//' -e 's/"$//'
}
# notEmpty directory
# Returns 0 if directory contains files/directories
notEmpty() {
if [ "`ls $1`" ]
then
return 0
else
return 1
fi
}
# dayDiff <date string 1> <date string 2>
# Prints number of days between the two
# Eg. dayDiff "Jan 10 2002" "Dec 28 1999"
# ==> 13
dayDiff() {
date -d "$1" > /dev/null 2>&1
if [ $? -ne 0 ]
then
#- option '-d' on date not supported. Give up.
echo "0"
else
year1=`echo "$1" | awk '{print $3}'`
year2=`echo "$2" | awk '{print $3}'`
day1=`date -d "$1" "+%j"`
day2=`date -d "$2" "+%j"`
nYears=`expr $year1 - $year2`
tmp1=`expr $nYears \* 365`
tmp2=`expr $day1 - $day2`
expr $tmp1 + $tmp2
fi
}
#dayDiff "`date '+%b %d %Y'`" "Dec 28 2001"
# getAllJobs jobInfoDirectory
# Prints list of all jobs in directory (e.g. runningJobs/)
# Also handles 'slaves' entries in jobInfo:
# slaves 1 ( penfold.23766 );
getAllJobs() {
if notEmpty $1
then
jobs=$1/*
for f in $jobs
do
line=`grep '^[ ]*slaves' $f 2>/dev/null`
if [ $? -eq 0 ]
then
slaveJobs=`echo "$line" | sed -e 's/.*(\(.*\)).*/\1/'`
jobs="$jobs $slaveJobs"
fi
done
else
jobs=''
fi
echo "$jobs"
}
# releaseLock jobId lockFile
# Releases lock on jobId
releaseLock () {
if [ -f $2 ]
then
#- move lock to finishedJobs
mv $2 $FOAM_JOB_DIR/finishedJobs/
fi
echo "Lock on job $1 released."
}
#-------------------------------------------------------------------------------
#
# Main
#
#-------------------------------------------------------------------------------
case "$1" in (-h | -help*) usage ;; esac
if [ $# -eq 1 ]
then
STATEFILE="$1"
elif [ $# -eq 0 ]
then
STATEFILE=${STATEFILE:-$DEFSTATEFILE}
else
usage
fi
#- Check a few things
if [ ! -d "$FOAM_JOB_DIR" ]
then
echo "$Script : directory does not exist."
echo " FOAM_JOB_DIR=$FOAM_JOB_DIR"
echo
if [ ! -d "$FOAM_JOB_DIR/runningJobs" -o ! -d "$FOAM_JOB_DIR/finishedJobs" ]
then
echo "$Script : invalid directory."
echo " FOAM_JOB_DIR=$FOAM_JOB_DIR"
echo
exit 1
fi
#- obtain rsh method
echo "Using remote shell type : $RSH"
echo ""
echo "Collecting information on jobs in"
echo " $FOAM_JOB_DIR"
echo ""
#- Collect machine names into $TMPFILE
# Also handles 'slaves' entry in jobInfo:
rm -f $TMPFILE; touch $TMPFILE
RUNJOBS=`getAllJobs $FOAM_JOB_DIR/runningJobs`
for f in $RUNJOBS
do
machinePid=`basename $f | sed -e 's/^"//' -e 's/"$//'`
machine=`echo "$machinePid" | sed -e 's/\.[0-9][0-9]*$//'`
pid=`echo "$machinePid" | sed -e 's/.*\.\([0-9][0-9]*\)$/\1/'`
fgrep "$machine" $TMPFILE >/dev/null 2>&1
if [ $? -ne 0 ]
then
echo "$machine" >> $TMPFILE
echo "Found machines:"
echo ""
#- Collect process info on all machines, one file per machine
mkdir -p $MACHDIR
cnt=1
while true
do
machine=`sed -n -e "${cnt}p" $TMPFILE`
if [ ! "$machine" ]
then
break
fi
machFile=$MACHDIR/$machine
rm -f $machFile
echo "Contacting $machine to collect process information:"
if [ $machine = `hostname` ]
then
echo " foamProcessInfo $machFile"
foamProcessInfo $machFile >/dev/null 2>&1
else
echo " $RSH $machine foamProcessInfo $machFile"
$RSH $machine foamProcessInfo $machFile >/dev/null 2>&1
fi
if [ $? -ne 0 -o ! -s $machFile ]
then
echo "** Failed collecting process information on $machine."
echo "Check $machFile and run foamProcessInfo by hand"
rm -f $machFile
else
echo "Succesfully collected information in $machFile ..."
fi
cnt=`expr $cnt + 1`
done
echo ""
#- Construct state for runningJobs; move non runnning jobs to finishedJobs
releaseAll=''
rm -f $STATEFILE
for f in $RUNJOBS
do
machinePid=`basename $f`
machine=`echo $machinePid | sed -e 's/\.[0-9][0-9]*$//'`
pid=`echo $machinePid | sed -e 's/.*\.\([0-9][0-9]*\)$/\1/'`
machFile=$MACHDIR/$machine
if [ -r $machFile ]
then
entry=`grep "^$pid " $machFile 2>/dev/null`
if [ $? -ne 0 -o ! "$entry" ]
then
if [ "$releaseAll" ]
then
releaseLock $machinePid $f
else
echo "Job $machinePid seems to be no longer running. Release lock? (y/a)\c"
if [ "${answ:-y}" = 'y' ]
then
releaseLock $machinePid $f
elif [ "${answ:-y}" = 'a' ]
then
releaseAll='yes'
releaseLock $machinePid $f
else
state='OTHR'
echo "$machinePid $state" >> $STATEFILE
fi
fi
else
state=`echo "$entry" | awk '{print $2}'`
echo "$machinePid $state" >> $STATEFILE
fi
fi
done
#- Collect old jobs in finishedJobs
OLDFILES=`find $FOAM_JOB_DIR/finishedJobs -mtime +$NDAYSLIMIT -print`
#- Construct state for finishedJobs and check on date of files.
if notEmpty $FOAM_JOB_DIR/finishedJobs
then
dateNow=`date '+%b %d %Y'`
for f in $FOAM_JOB_DIR/finishedJobs/*
do
sz=`ls -s $f | awk '{print $1}'`
if [ "$sz" -gt 0 ]
then
machinePid=`basename $f`
machine=`echo $machinePid | sed -e 's/\.[0-9][0-9]*$//'`
pid=`echo $machinePid | sed -e 's/.*\.\([0-9][0-9]*\)$/\1/'`
end=`getEntry $f endDate`
if [ ! "$end" ]
then
state='ABRT'
else
nDaysOld=`dayDiff "$dateNow" "$end"`
if [ "$nDaysOld" -gt $NDAYSLIMIT ]
then
OLDFILES="$OLDFILES $f"
fi
state='FINI'
fi
echo "$machinePid $state" >> $STATEFILE
fi
done
fi
#- Remove old locks
nOldFiles=`echo "$OLDFILES" | wc -w`
if [ "$nOldFiles" -gt 0 ]
then
echo "You seem to have $nOldFiles locks older than $NDAYSLIMIT days in finishedJobs/"
$ECHO "Do you want to remove these? (y)\c"
read answ
if [ "${answ:-y}" = 'y' ]
then
rm -f $OLDFILES
fi
fi
rm -f $TMPFILE
rm -r $MACHDIR
echo ""
echo "Updated stateFile:"
echo " $STATEFILE"
echo ""
#------------------------------------------------------------------------------