#!/bin/sh
#------------------------------------------------------------------------------
# =========                 |
# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
#  \\    /   O peration     |
#   \\  /    A nd           | Copyright (C) 2011-2015 OpenFOAM Foundation
#    \\/     M anipulation  | Copyright (C) 2018 OpenCFD Ltd.
#-------------------------------------------------------------------------------
# License
#     This file is part of OpenFOAM.
#
#     OpenFOAM is free software: you can redistribute it and/or modify it
#     under the terms of the GNU General Public License as published by
#     the Free Software Foundation, either version 3 of the License, or
#     (at your option) any later version.
#
#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
#     for more details.
#
#     You should have received a copy of the GNU General Public License
#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
#
# Script
#     foamJob
#
# Description
#     Run an OpenFOAM job in background.
#     Redirects the output to 'log' in the case directory.
#
#------------------------------------------------------------------------------
usage() {
    exec 1>&2
    while [ "$#" -ge 1 ]; do echo "$1"; shift; done
    cat<<USAGE

Usage: ${0##*/} [OPTION] <application> ...
options:
  -case <dir>       specify alternative case directory, default is the cwd
  -parallel         parallel run of processors
  -screen           also sends output to screen
  -append           append to log file instead of overwriting it
  -wait             wait for execution to complete (when not using -screen)
  -version=VER      specify an alternative OpenFOAM version
  -help             print the usage

* run an OpenFOAM job in background.
  Redirects the output to 'log' in the case directory

USAGE
    exit 1
}

# Echo strings that have single quotes
echoArgs() {
    addSpace=""

    for stringItem in "$@"
    do
        echo -n "${addSpace}"

        if [ "${stringItem##* }" = "$stringItem" ]
        then
            echo -n "$stringItem"
            addSpace=" "
        else
            echo -n "'$stringItem'"
            addSpace=" "
        fi
    done

    unset stringItem addSpace
}


# Replacement for possibly buggy 'which'
findExec() {
    case "$1" in
    */*)
        if [ -x "$1" ]
        then
            echo "$1"
            return 0
        fi
        ;;
    esac

    oldIFS=$IFS
    IFS=':'
    for d in $PATH
    do
        # echo "testing: $d/$1" 1>&2
        if [ -x "$d/$1" -a ! -d "$d/$1" ]
        then
            # echo "Found exec: $d/$1" 1>&2
            IFS=$oldIFS
            echo "$d/$1"
            return 0
        fi
     done
     IFS=$oldIFS
     echo ""
     return 1
}



# Main script
#~~~~~~~~~~~~
unset parallelOpt screenOpt waitOpt
unset projectVersion

# Parse options
while [ "$#" -gt 0 ]
do
    case "$1" in
    -h | -help*)
        usage
        ;;
    -case)
        [ "$#" -ge 2 ] || usage "'$1' option requires an argument"
        cd "$2" 2>/dev/null || usage "directory does not exist:  '$2'"
        shift
        ;;
    -p | -parallel)
        parallelOpt=true
        ;;
    -s | -screen)
        screenOpt=true
        ;;
    -a | -append)
        appendOpt=true
        ;;
    -w | -wait)
        waitOpt=true
        ;;
    -version=*)
        projectVersion="${1#*=}"    # for foamExec
        ;;
    -v | -version)
        [ "$#" -ge 2 ] || usage "'$1' option requires an argument"
        projectVersion="$2"         # for foamExec
        shift
        ;;
    --)
        shift
        break
       ;;
    -*)
        usage "invalid option '$1'"
        ;;
    *)
        break
        ;;
    esac
    shift
done

# ------------------------------------------------------------------------------

[ "$#" -ge 1 ] || usage "No application specified"

# The requested application
appName="$1"

# Use foamExec for a specified version
# Also need foamExec for remote (parallel) runs
if [ -n "$projectVersion" -o "$parallelOpt" = true ]
then
    # When possible, determine if application even exists
    if [ -z "$projectVersion" ]
    then
        findExec "$appName" >/dev/null || usage "Application '$appName' not found"
    fi

    # Use foamExec for dispatching
    APPLICATION=$(findExec foamExec) || usage "'foamExec' not found"

    [ -n "$projectVersion" ] && APPLICATION="$APPLICATION -version $projectVersion"

else
    APPLICATION=$(findExec "$appName") || usage "Application '$appName' not found"
    echo "Application : $appName"
    shift
fi


if [ "$parallelOpt" = true ]
then
    # parallel
    # ~~~~~~~~

    #
    # Check if the case decomposed
    #
    if [ -r "processor0" -o -r "processors" ]
    then
        nprocs="$(foamDictionary -entry numberOfSubdomains -value system/decomposeParDict 2>/dev/null)"
    else
        echo "Case is not currently decomposed"
        if [ -r system/decomposeParDict ]
        then
            echo "system/decomposeParDict exists"
            echo "Try decomposing with \"foamJob decomposePar\""
            exit 1
        else
            echo "Cannot find system/decomposeParDict file required to decompose the case for parallel running."
            echo "Please consult the User Guide for details of parallel running"
            exit 1
        fi
    fi

    #
    # Find mpirun
    #
    mpirun=$(findExec mpirun) || usage "'mpirun' not found"
    mpiopts="-np $nprocs"

    #
    # Check if the machine ready to run parallel
    #
    echo "Parallel processing using $WM_MPLIB with $nprocs processors"
    case "$WM_MPLIB" in
    *OPENMPI)
        # Add hostfile info
        for hostfile in \
            hostfile \
            machines \
            system/hostfile \
            system/machines \
            ;
        do
            if [ -r $hostfile ]
            then
                mpiopts="$mpiopts -hostfile $hostfile"
                break
            fi
        done

        # Send FOAM_SETTINGS to parallel processes, so that the proper
        # definitions are sent as well.
        mpiopts="$mpiopts -x FOAM_SETTINGS"
        ;;
    esac

    #
    # Run (in parallel)
    #
    if [ "$screenOpt" = true ]
    then
        [ "$appendOpt" = true ] && teeOpts=" -a"
        echo "Executing: $mpirun $mpiopts $APPLICATION $(echoArgs "$@") -parallel | tee $teeOpts log"
        $mpirun $mpiopts $APPLICATION "$@" -parallel | tee $teeOpts log
    else
        if [ "$appendOpt" = true ]
        then
            echo "Executing: $mpirun $mpiopts $APPLICATION $(echoArgs "$@") -parallel >> log 2>&1"
            $mpirun $mpiopts $APPLICATION "$@" -parallel >> log 2>&1 &
        else
            echo "Executing: $mpirun $mpiopts $APPLICATION $(echoArgs "$@") -parallel > log 2>&1"
            $mpirun $mpiopts $APPLICATION "$@" -parallel > log 2>&1 &
        fi

        pid=$!
        if [ "$waitOpt" = true ]
        then
            wait $pid
        fi
    fi

else
    #
    # Run (on single processor)
    #
    if [ "$screenOpt" = true ]
    then
        [ "$appendOpt" = true ] && teeOpts=" -a"
        echo "Executing: $APPLICATION $(echoArgs "$@") | tee $teeOpts log &"
        $APPLICATION "$@" | tee $teeOpts log &
        wait $!
    else
        if [ "$appendOpt" = true ]
        then
            echo "Executing: $APPLICATION $(echoArgs "$@") >> log 2>&1 &"
            $APPLICATION "$@" >> log 2>&1 &
        else
            echo "Executing: $APPLICATION $(echoArgs "$@") > log 2>&1 &"
            $APPLICATION "$@" > log 2>&1 &
        fi

        pid=$!
        if [ "$waitOpt" = true ]
        then
            wait $pid
        fi
    fi
fi


#------------------------------------------------------------------------------