#!/bin/sh
# $Id: benchmark.sh 975 2011-03-29 10:44:17Z gsutre $

#
# Copyright (c) 2009, 2010, 2011 CNRS & Université Bordeaux 1.
#
# Author(s): Alexander Heußner <alexander.heussner@labri.fr>
#            Grégoire Sutre <gregoire.sutre@labri.fr>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#


# Configuration Variables ######################################################

# Debugging messages are disabled by default.
DEBUG=

# McScM base directory (this script should be in the test sub-directory).
BASE_DIR=$( dirname $0 )/..

# Model-checker executable to use.  Replace with mcscm.native.static if needed.
CHECKER="${BASE_DIR}/bin/mcscm.native"

# Directory containing scm examples.
EXAMPLE_DIR="${BASE_DIR}/examples/scm"

# Examples to benchmark on (<filename>:<description>:<extra options>).
EXAMPLES="
ABP:Alternating Bit Protocol
con_disc_reg:Connection/Disconnection Protocol
Nested:Nested Connection/Disconnection Protocol
NonRegular:Non-Regular Protocol
tcp_simplest:Simplified TCP
simple_server:TCP-based Protocol With 2 Clients:-extrapol-start-param -1
ring:Token Ring Protocol
peterson_3:Peterson's Leader Election Algorithm
sliding:Sliding Window Protocol
brp_like:Bounded Retransmission Protocol
pop3:Post Office Protocol"

# Default list of (partition) refinement methods to try on each example.
REFINEMENTS="apinv-bwd apinv-fwd upinv-bwd upinv-fwd"

# Default timeout in seconds (max CPU time) for each model-checking run.
TIMEOUT=900

# Default options used for all examples.
OPTIONS="-statistics -screen-width 72 \
-no-verification -no-scm-consistency-check \
-extrapolation bisim-fwd -extrapol-coloring final -graph-exploration fwd-bfs"


# Command-line Processing ######################################################

usage ()
{
    cat <<EOF
Usage: `basename $0` [options]

  -c <exe>    Use model-checker executable <exe>.
  -d          Print debugging messages.
  -e <eng>    Set the model-checking engine to <eng>.
  -h          Print this message and exit.
  -l          Make all channels lossy.
  -r <ref>    Try refinement methods in the space-separated list <ref>.
  -t <int>    Set the (CPU) timeout to <int> for each bench.
  -x <opt>    Append <opt> to the options passed to the McScM executable.
EOF
}

while getopts "c:de:hlr:t:x:" option; do
    case $option in
	c)
	    CHECKER="$OPTARG"
	    ;;
	d)
	    DEBUG="yes"
	    ;;
	e)
	    OPTIONS="$OPTIONS -mc-engine $OPTARG"
	    ;;
	h)
	    usage
	    exit 0
	    ;;
	l)
	    OPTIONS="$OPTIONS -channel-policy all-lossy"
	    ;;
	r)
	    REFINEMENTS="$OPTARG"
	    ;;
	t)
	    TIMEOUT="$OPTARG"
	    ;;
	x)
	    OPTIONS="$OPTIONS $OPTARG"
	    ;;
	*)
	    usage
	    exit 1
	    ;;
    esac
done

if [ "$#" -ge "$OPTIND" ]; then
    usage
    exit 1
fi


# Auto-configuration and checks ################################################

# Check that the model-checker executable exists.
if [ ! -x "${CHECKER}" ]; then
    echo "Couldn't find model-checker executable \`${CHECKER}'.  Aborting."
    exit 1
fi

# Check that the scm example directory exists.
if [ ! -d "${EXAMPLE_DIR}" ]; then
    echo "Couldn't find example directory \`${EXAMPLE_DIR}'.  Aborting."
    exit 1
fi

# Command-line used to run the tool on each example
CMD="${CHECKER} ${OPTIONS}"


# Main #########################################################################

echo "Simple Benchmark Script for McScM (${CHECKER})"
echo "Global McScM options used for all examples: ${OPTIONS}"
echo "Timeout for each model-checking run: ${TIMEOUT} (s)"

IFS="
"

for e in $EXAMPLES; do
    IFS=":"
    set -- $e
    file="$1.scm"
    description="$2"
    opt="$3"
    unset IFS

    echo
    echo "==================================================================="
    echo "${description}   (${file})"
    echo "Specific options for this example: ${opt}"

    for o in $REFINEMENTS; do
	cmdline="${CMD} ${opt} -partition-refinement $o ${EXAMPLE_DIR}/${file}"
	echo "-------------------------------------------------------------------"
	echo "With partition-refinement: $o"
	echo
	if [ "$DEBUG" = "yes" ]; then
	    echo "Command line: ${cmdline}"
	    echo
	fi
	ulimit -t ${TIMEOUT}
	if ! ${cmdline}; then
	    echo
	    echo "!!!! McScM returned an exit status of 1 (false) or was killed."
	    echo "!!!! Note: CPU time was limited to ${TIMEOUT} seconds."
	    echo
	fi
    done
done
