#!/bin/sh
# $Id: benchmark.sh 1269 2011-10-19 23:13:22Z gsutre $

#
# Copyright (c) 2009, 2010, 2011 CNRS & Université Bordeaux 1.
#
# Author(s): Alexander Heußner <alexander.heussner@labri.fr>
#            Grégoire Sutre <gregoire.sutre@labri.fr>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#

self="$(basename "$0")"


# Constant Parameters ##########################################################

# Benchmark's complete example set (<filename>:<description>:<extra options>).
EXAMPLE_SET="
ABP:Alternating Bit Protocol
brp_like:Bounded Retransmission Protocol
con_disc_reg:Connection/Disconnection Protocol
load_balancing:Simple Asynchronous Load Balancer
load_balancing_err:Simple Asynchronous Load Balancer (with error)
Nested:Nested Connection/Disconnection Protocol
NonRegular:Non-Regular Protocol
peterson_3:Peterson's Leader Election Algorithm
pop3:Post Office Protocol
ring:Token Ring Protocol
simple_server:TCP-based Protocol With 2 Clients
sliding:Sliding Window Protocol
tcp_simplest:Simplified TCP
tcp_simplest_err:Simplified TCP (with error)"


# Configuration Variables ######################################################

# Make sure that IFS is unset.
unset IFS

# Debugging messages are disabled by default.
DEBUG=

# McScM base directory (this script should be in the test sub-directory).
BASE_DIR=$( dirname $0 )/..

# Model-checker executable to use.  Replace with verify.native.static if needed.
CHECKER="${BASE_DIR}/bin/verify.native"

# Directory containing scm examples.
EXAMPLE_DIR="${BASE_DIR}/examples/scm"

# Default list of example names to benchmark on (empty means all examples).
EXAMPLES=

# Default list of model-checking engines to try on each example.
MC_ENGINES="absint, armc, cegar(_), lart(_)"

# Default list of trace-checking engines (used for `_' in MC_ENGINES).
TC_ENGINES="apinv-bwd, apinv-fwd, upinv-bwd, upinv-fwd"

# Default timeout in seconds (max CPU time) for each model-checking run.
TIMEOUT=900

# Default options used for all examples.
OPTIONS="-statistics -columns 72 \
-no-validation -no-scm-consistency-check \
-extrapolation bisim-fwd -extrapol-coloring auto \
-graph-exploration fwd-bfs -tree-exploration bfs"


# Helper functions and constants ###############################################

# Constant variable to hold the newline character.
newline="
"

error ()
{
    echo "$self: error: $1" 1>&2
}

#
# Removes spaces, tabs and newlines at the start and end of the given word.
#
trim ()
{
    result=$1
    t=${result%%[! 	$newline]*}
    result=${result#"$t"}
    t=${result##*[! 	$newline]}
    result=${result%"$t"}
    echo "$result"
    unset result
}

#
# Checks that the built-in shell variable IFS  is unset.
#
check_IFS_unset ()
{
    if [ "${IFS+set}" = "set" ]; then
	error "IFS is set."
	return 1
    fi
}

#
# Returns the list of all example names, separated by the given separator.
#
print_example_set ()
{
    sep=$1

    check_IFS_unset || return $?

    unset result
    IFS=${newline}
    for e in ${EXAMPLE_SET}; do
	if [ -n "${result}" ]; then
	    result="${result}$sep"
	fi
	result="${result}${e%%:*}"
    done
    echo "$result"
    unset IFS
}

#
# Finds a given example name in EXAMPLE_SET, and returns the matching entry
# (or "" if the name is not found).
# Precondition: IFS must be unset.
#
find_example ()
{
    check_IFS_unset || return $?

    IFS=${newline}
    for e in ${EXAMPLE_SET}; do
	if [ "$1" = "${e%%:*}" ]; then
	    echo "$e"
	    break
	fi
    done
    unset IFS
}

#
# Returns the list of example entries corresponding to the given list of example
# names.  The input list is comma-separated (see option -f), but the output list
# is newline-separated (as in EXAMPLE_SET).
# Precondition: IFS must be unset.
#
example_subset ()
{
    exlist=$1

    check_IFS_unset || return $?

    unset result
    IFS=','
    for n in ${exlist}; do
	unset IFS
	n=$(trim "$n")
	e=$(find_example "$n") || return $?
	if [ -z "$e" ]; then
	    error "unknown example name \`$n'."
	    exset=$(print_example_set ", ") || return $?
	    error "please choose among: ${exset}."
	    return 1
	fi
	result="${result}${newline}$e"
    done
    echo "$result"
    unset IFS
}

#
# Generates a newline-separated list of mc:tc entries from the given lists of
# model-checking (mc) engines and trace-checking (tc) engines.  The `_' wildcard
# in mc entries is expanded to all tc entries.
# Precondition: IFS must be unset.
#
mctc_list ()
{
    mclist=$1
    tclist=$2

    check_IFS_unset || return $?

    unset result
    IFS=','
    for o in ${mclist}; do
	unset IFS
	o=$(trim "$o")
	if [ -z "${o%%*[()]*}" ]; then
	    # There is a parenthesis.
	    m=${o%%(*}
	    t=${o##*(}
	    t=${t%%)*}

	    if [ "$o" != "$m($t)" ]; then
		error "invalid model-checking enging specification \`$o'."
		return 1
	    fi

	    m=$(trim "$m")
	    t=$(trim "$t")

	    # Expand $t if it is the wildcard character.
	    if [ "$t" != "_" ]; then
		result="${result}${newline}$m:$t"
	    else
		IFS=','
		for t in ${tclist}; do
		    unset IFS
		    t=$(trim "$t")
		    result="${result}${newline}$m:$t"
		done
	    fi
	else
	    # There is no parenthesis.
	    result="${result}${newline}$o"
	fi
    done
    echo "$result"
    unset IFS
}


# Command-line Processing ######################################################

usage ()
{
    cat <<EOF
Usage: $self [options]

  -c <exe>    Use model-checker executable <exe>.
  -d          Print debugging messages.
  -e <lst>    Comma-separated list of model-checking engines.
  -f <lst>    Comma-separated list of example names.
  -h          Print this message and exit.
  -l          Make all channels lossy.
  -r <lst>    Comma-separated list of trace-checking engines.
  -t <int>    Set the (CPU) timeout to <int> for each bench.
  -x <opt>    Append <opt> to the options passed to the McScM executable.
EOF
}

while getopts "c:de:f:hlr:t:x:" option; do
    case $option in
	c)
	    CHECKER="$OPTARG"
	    ;;
	d)
	    DEBUG="yes"
	    ;;
	e)
	    MC_ENGINES="$OPTARG"
	    ;;
	f)
	    EXAMPLES="$OPTARG"
	    ;;
	h)
	    usage
	    exit 0
	    ;;
	l)
	    OPTIONS="$OPTIONS -channel-policy all-lossy"
	    ;;
	r)
	    TC_ENGINES="$OPTARG"
	    ;;
	t)
	    TIMEOUT="$OPTARG"
	    ;;
	x)
	    OPTIONS="$OPTIONS $OPTARG"
	    ;;
	*)
	    usage
	    exit 1
	    ;;
    esac
done

if [ "$#" -ge "$OPTIND" ]; then
    usage
    exit 1
fi


# Auto-configuration and checks ################################################

# Check that the model-checker executable exists.
if [ ! -x "${CHECKER}" ]; then
    error "couldn't find model-checker executable \`${CHECKER}'."
    exit 1
fi

# Check that the scm example directory exists.
if [ ! -d "${EXAMPLE_DIR}" ]; then
    error "couldn't find example directory \`${EXAMPLE_DIR}'."
    exit 1
fi

# Command-line used to run the tool on each example.
CMD="${CHECKER} ${OPTIONS}"

# Generate list of example entries.
if [ -z "${EXAMPLES}" ]; then
    EXAMPLES="${EXAMPLE_SET}"
else
    EXAMPLES="$(example_subset "${EXAMPLES}")" || exit $?
fi

# Generate list of model-checker:trace-checker entries.
MC_ENGINES="$(mctc_list "${MC_ENGINES}" "${TC_ENGINES}")" || exit $?


# Main #########################################################################

echo "Simple Benchmark Script for McScM (${CHECKER})"
echo "Global McScM options used for all examples: ${OPTIONS}"
echo "Timeout for each model-checking run: ${TIMEOUT} (s)"

IFS=${newline}
for e in ${EXAMPLES}; do
    # Parse this example entry
    IFS=':'
    set -- $e
    file="$1.scm"
    description="$2"
    specopt="$3"
    set --
    unset IFS

    echo
    echo "==================================================================="
    echo "${description}   (${file})"
    echo "Specific options for this example: ${specopt}"

    IFS=${newline}
    for o in ${MC_ENGINES}; do
	opt="${specopt}"

	# Parse this model-checker:trace-checker entry
	IFS=':'
	set -- $o
	m="$1"
	t="$2"
	set --
	unset IFS

	echo "-------------------------------------------------------------------"
	opt="${opt} -mc-engine $m"
	echo "With model-checking engine: $m"
	if [ -n "$t" ]; then
	    opt="${opt} -tc-engine $t"
	    echo "With trace-checking engine: $t"
	fi
	cmdline="${CMD} ${opt} ${EXAMPLE_DIR}/${file}"
	echo
	if [ "${DEBUG}" = "yes" ]; then
	    echo "Command line: ${cmdline}"
	    echo
	fi
	ulimit -t ${TIMEOUT}
	if ! ${cmdline}; then
	    echo
	    echo "!!!! McScM returned an exit status of 1 (false) or was killed."
	    echo "!!!! Note: CPU time was limited to ${TIMEOUT} seconds."
	    echo
	fi
    done
done
